diff --git a/.gitattributes b/.gitattributes index e9077267405898600770c000fb156cbd0d10c12b..1e200d8fe2278920e36df1a4cd45bca5fab5e571 100644 --- a/.gitattributes +++ b/.gitattributes @@ -393,3 +393,75 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text 146m14b1b5/evaluation/generation/examples.146m14b1b5_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text 421m3b91b5/evaluation/generation/examples.421m3b91b5_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text 421m3b91b5/evaluation/generation/examples.421m3b91b5_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text +1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text +619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e521fbcb2b02b4d874715dcb98330aaa47ee4e9 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db1b094cf354e45a778a43db7f54a08f1e27d2634317ff51ef378e3b3838d11d +size 205568023 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fdb0f16d33d8bd36fba6f39acbd94659bfa4cb4 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e14beb359551b58199f235c0e02bd941c30fbdb5a9b91fc482c7f38aad63aab +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b1db8399b5815f6225cde0262f6ab4c2651fae0 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181714819a3d922237ce56d03a83cf40b5f4eea572c5e4ce440fc7a4c48ca3f8 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff03f5a325b1cdcdf7e3f7b14b08c2f72a92f632 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f911b2985fb7aa44b8d9b35996cbbbd13169d82415f52e507b8022375b28cb1 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7dc93a77880b2953cc797f513645df54be82e11f --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2061299a76f33576b9ce250b9e0bf970feb95fe3fd646c08b0ca2622acb946c +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad7b5cb16ca5e2c66f15fa9aca3b7cfb250d5e93 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d57cb598fc1846d6f855d6e5c4e00e82797f1dc13ba2408284b31d1868b07ad5 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f3461d30470c344c27151d0c44ef73ece20e688 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be8e2e530d4ce9483e22589a1fd6dd09da81e41dc24d64a852038309a8cce56e +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b2d8a6efa32b743718f64ab2133280945cc3e4a --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1521ee4f6ef261af3bfab5f68be1aa6e728d19afec0ec2b0f0d86df0bb29b052 +size 205567970 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47072e45fa5d522f9de39111f128eb1be87b2f8e --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ebf2a06964504a79969c0b6658aed95cab5640c83ebbcf7a071e49d30a6492 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6d08416c07850055315aef2292f25b3ad58fefe --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:698e822cc2f181e5068a1648170148bd2ae1e3c1a5e12c578f4829ccd36b245a +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8e497c9fe984b590a3c701dbda1ad400e44c0ce --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a879c0b44a323db02107b32bd7b87242a9708b0bc569e8e0a6897be4d9afc12d +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..714cffb97b3155096424ba73f480646016d7c9f5 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:230d74416f2a36010b0aeed8054e0eee345c5a9449b7196bee1c2f4d78f3429f +size 205568151 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bf09b51857f4fc88f957c59256ce087474fd8b0 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4788c2eb4698566c1676159a357bedaed8a3d881e6f1ba82b48d1cb3b3d46179 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd548e2661ae96801bb8e021a9547c5bf9e9eb8a --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93e3c08b316a92d5ac6afb52cfe3d2f373bab080dda19b2e19777f5a6cc589d6 +size 205568034 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4113d5b6c2ba7ef65eae50c5efd01978033b670c --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de8057323ade7890ef81f60f2b74ac6e8d5ac3ae468705ee7edf5de9545f5027 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..700147cd9b3948f902c8827ccbeebf60a8d7e2ac --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c88bf0318aeeb0bb96704665ae829eba1e6dda41d8d85ff5fb7d6cba18e43f9 +size 205568098 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca13e7d0e011f6ce5048a85c0bb04c034ee7389a --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb032cfe2338ef0eb0a99a66d0391d473838369aeb587c2148d56d4238ad2000 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5fe0d38c36d23782c73a0c4dd771085472ab205 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf3795a1825455fa6a229b087838e7fd255e08e53530c1ff50e434b5ba9b583 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac59c5178239fbf71a2d2fda2969612510b96f73 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1918b9582f6faf60861c22692cb394cfc42bac80d9b8508f117a6ba18d3fd56d +size 205568034 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e93788ac236f1b83ff0e30e9b3914296a745224f --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b877a352589c848a7f208966768ff31cf32c6cc2fee012c7b7404c19fd8ec5d4 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bc0d4f6d13bb8834fbf5c5eb96836cbe5c6a79e --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad6a10f00b3d074e170bc908f518747d8e2fe08ba7207514ab74247983db776 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3ef61264400e8ad1dd4705af20bcf896923d32b --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1805cf8704c5ab3abe7465a2c03ea6ced18434217c8da107d6a73dc7543f3561 +size 205568098 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..069537b87f606c205029c1ee43e838049952a10e --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4f4918f1bfd75d87896b4fbc003421a8a416cd7a3fcdb0e6cbca0b41f4278f2 +size 205568151 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b611111fea45d2b596e1af0f3b3d835aba42955 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc838be72907974b7ac55a792e9b3a2afa2d6216764b0dcb3a7a89c2aa0c40e2 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcb9c78b38e6b6ef32fd9e05b9819f4fb47b20f4 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3e3fbf16adb55af4e1c5d3ece6da8e5b84c49e4ecd7714b8238abc6e3cf96b9 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0885a281dee5b80ca16f2d70978ba36c385e7e04 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc4b6c277db81f6b3c3ec4482357330c64947429ad8636d20a1fb51af9af9c8 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..792c3c3c05f0dd7bfb4870b839d3ac1849c553f6 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c5125b823d77fdfa1d25ac61d6c9471443e12944ea4ee109452318b7514e8f +size 205568034 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37836b6409485c0ef2646ba489c1de1fbace28f8 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d548dd512dbfbb5cfdd7d167a0b9c71a2bf6371e8c83c58f42f70f1a4faa445c +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52bd4d35861468dee446a93ce069e9a140dd9bfc --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4b610733a650e04421cbb5ce99ad503102274b06bdd6dbfbc2db53d40578b6a +size 205568290 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51bbac3ede32c20a58b6fe52852b446713ad0fe2 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21474a01f5d09044034f6671f057885aa197ab638d35112507ab55a3163cf8fd +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aea73a7e858725d7812af531645ed6a45a534070 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:101efdd5ec18eff9549a4e35bb4b82efc7c26029cf08c0628ec157f2ca6b1243 +size 205568098 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5be65a62786993bb2c25a7d4fa9c7b448925022d --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d6abfd051c4be40295a61e6bfdba7b5bf5c1ee82c5d652c78707259dbca83fd +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e36408ebdcbffdaa7605fc52b3313d5db4120598 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3baa12e7b6b33eac3a6b9c40756ff55b7b3f7fdcd890183f6ccbcc8293a8d3d +size 205568098 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2d771b9a8a9f3e17e5dd8677fa740e64322be9a --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ea882bebc763dcdba7504b14ee9b7dfd798b89749f255c6fe745a6e3597131e +size 205568215 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1967e0db7ebdb5bac31412ef73132d8575403550 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e0f0d104a3aa0059f55d9d758160ef829fdf22a30173bd03e1048d60426940a +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e39d6d3d3a515f47baf229a1974dcf3ec84ca5b7 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03c0e65929e60310f9ad4ce7134a844591e29e42dcba804b0965571bc66af6f2 +size 205568354 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e982512ff22d6a09783e8102533e07b85fae4cf5 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c1ce2a4ec7163259fa982a6d28c4ed53cbee68d2ed40361302bde7f85e44d4b +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5948a43ab2a672f764a00aa5fd7bb1537def07c3 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25008365654f9c0982e459a840825712eba51106eebcf18db7236f6e0586969e +size 205568034 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9ebdca2684aede49a2ede9d1444285dcaf3674a --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e86de698fc9290f7915c8a205e1462457cf6f93df4c17825c91bd35bae100a8a +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11771a84f5101c893c0c50c59334cdbfa5a2634e --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:231394627d622c2e6f306ac4507d6834f0e9fc7cbffc8446c0d9de46ee270559 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1581d5c2240dc5a6772b21df5a5dac4d4368b220 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44c0991da1a6173b4e6fb3b1f9a2434f3d7fb64a9a2b67cc1cd2c629269c0f21 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d090ee34cb780a6060dd03a03c4207e2dde4ab55 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0252ba4e649f65ab4196550ec8e794112f27f588c50031a386741977325efee2 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9be98caecdf5bc6950995ed510044b0d49ad0ac7 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f637ec37b2c19c5071d4b5e40b4d0fb3e49d094b6a696eaef4e94d3cf714e7d +size 205568034 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f71fe4cd1d75cf271efc9420aa71b1a0fdc7a0e --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d2510160a1ec149a69c9c280adc4581c89182cdc49dbd8a953ede05c05529e +size 205568290 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8076ad7b2e86a118dfd2b63db437e0ed5a496d4b --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3508e9fe28d6d7c837a2cb7ad9fbab8804a060321951fcdffa8929ac67bc0d4c +size 205568023 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ec6b6c9b2daf8a36165e2b98b0ea9ec7f0ad4a2 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6e4a38516b7e2ba816b6f113649b05b0a7aaacd07feae7afbf74bcfb4f5f284 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74a3e181d5d2adfc980c10693a30ced10880d332 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6811f003a404942555437e84a0a7de89052aeb0316d0ca5b57f3f8f8f97b7d0f +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89034b1cc0620fbb526786340868bf7ae3432aad --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dda45e2e1ee6d6e285503292c83eb4d0fd932e1ceb3145ed13edb8a6a31a70b +size 205568290 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3c591bf8acb9b715388982af595a92102ff6408 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d83856e3a620a2113c36c4879745b947efa57508e6accb0a8815403645ef087 +size 205568034 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56918d371ad773be1cb43076c63fd39947e807a1 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd67d72d1d771001be1df85cf3176764ea219900bca72443c428236f6f35a3b0 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e27a4b821f19fb0e4c71a38dfdb788c3cb1a94d4 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ed201d7c8a41c93ce4e52c38f707288955540162b690d67424378f6b6e9cb6 +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..270e02d6e115f0832371d15ad31d785e10b0f804 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b35d94d938e088c5982e587c4be33c3f7731b0336fc93c2ef4536b29f928434f +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfef9e21a58ecd0dbe9792ad06ea17f2c3f64a21 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f3b655edee2b3373f161559acca9f0c1c8c719903216ca644df85b742c71d62 +size 205568290 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef956fb2c36792b4b6478bb56c97c12945629bcc --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5813d4ef6619796b3790e01c8db87f8e1e69e2054688ead709a86dea13eb50ea +size 205568098 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..934e9d2417e8d7664d15756b12174223902e7214 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17f89b5f68dc8c9d8a7f1b82f3ade21dd8fcbaef592b52ba82810a933ff4362 +size 205568226 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d96fbe6a83d407d0445a109f62433c2f40f60a8 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d04b5baf250bc6eaabced5f1a9df3839a828eeef7040ae05cbc4f314e6f5c6b +size 205568023 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..503a9e822b3dd44b02bd5147f0b835d2221cf817 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a46eaaddf2c67338a4a95b0403be6802b0ba05f000865094de7f756c7d6f38a +size 205568162 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbb3f9c188b93260b69a7a49a29764e2c2266df5 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34ce977c8edd1c8ab46878fa0b75a8f847d1696129c5dd863f106f82d75d759f +size 205568098 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2c7bd56b3ffe27ec112ae422ab19fff23c26e65 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d2b5fd16bd7d207c3b881c7e2a7aafd2f3a7486c4703b03dc0ff11d2e3e48f2 +size 205568290 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4433a387cc1eb465a4e8a34a40ae254e22d08c76 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5d2983dd8438b123c9c7f54ce5382f42f0ca9cde6089b26523855349ffaea0c +size 205568034 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43b88c5c620a9b7b4ea8b2ddf63cbe3e52f1e95c --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a4b431c8c88b729528034ce81cc28434be7ef7d248473f6d870b0f8915df1a5 +size 205568151 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbbcea37107a1a9888614515dad1b1f4dabb6959 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df48c13e728e5fdf0d531e579df8a78173d6f7498f8a6b48a0ceac1c5fd30c6e +size 205568151 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3358d4549b713189e103a56763f05d4595f7c11 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f28db1ae0fef7cb3ded8a3516acdb3a9f61ca0deb0e061cd3c8bcbaf08b089b0 +size 205568151 diff --git a/1b11b5100m/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/1b11b5100m/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89f2e788f740bbf7241a83e93ca67973ae526898 --- /dev/null +++ b/1b11b5100m/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b48c8bd51d4898731b7d5754ec6ce3f0e395494f18ea3bb5728fbefc067bf48 +size 205568151 diff --git a/1b11b5100m/global_step2891/layer_01-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9fb52af99764ab943b099b964070f5ab29d8ebc --- /dev/null +++ b/1b11b5100m/global_step2891/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0abd38754965079ced96a9996f7da199c2e2ac170f0a3b5a23575451d93d8b20 +size 187630851 diff --git a/1b11b5100m/global_step2891/layer_03-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69b0ada43afc9d50b425cfa463d438673f3a5f46 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b46d3af64319b3c3254c4c5295dccfdb190be1b525715f1d98044fa7785de82 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_04-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afc14b2088b5c144fcfd2a5480b7c7d67aee88f9 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f03309e7e39b6502fd96f593378b57b61572f63015d98bf76c751ea0f073fb +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_05-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90185ccff942fb578017a37483b2b7386c0d1bc8 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08dbee04a910796da52347d7c39eeaf80e221d6141f22b03d47c250ff709e20d +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_06-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b83ba3f9f00862119e1baad0b23abf8627577d9a --- /dev/null +++ b/1b11b5100m/global_step2891/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d171a96afa033cfee5e173b24228bd9106ac82a10f33030a0e71b65f54a6acae +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_07-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f47f64ffa5f40df8b44207e5785c41c5a536e94 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f9c6fb28d833c69cc6df457c18a5d6641f21d9be6b1db7286ce25cb994162a4 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_08-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fb2a52bf77e3b1b786215d9c30c1f800236f05f --- /dev/null +++ b/1b11b5100m/global_step2891/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3222f5337d616972504486d896951295b5645a7e4657b83dde66a34e5b6584c +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_09-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a34403184100231b92fc827901667506cdd1365e --- /dev/null +++ b/1b11b5100m/global_step2891/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b59ac9bab214f6f0761bd6856ab297e083a0f346cd965bbabebc7d2a0a4f5ae0 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_10-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df9ce21b44ab14106a91dae1fe8875bd05865ffd --- /dev/null +++ b/1b11b5100m/global_step2891/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0da668db28ff52da2e52f5524ba3af37c748af01d10090be4dfa63d39526a47a +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_11-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97b4273e6087f023edcbfd3939da1cc7d174a5c8 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:338ebf2c759bcefdc1b57522b4f224e3076a001400a930515fcf26fe8d5de51b +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_12-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..337f13f46d78eb380d4e4c28ebae12331b98a75b --- /dev/null +++ b/1b11b5100m/global_step2891/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c352df20035d60f3de43c7abcce6878bfc1acea5df99f3951f261cce97681cc9 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_13-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..693c8d2c3e30f03fc1a0570ec54f4b7eebab3f26 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41b2afcb652687c8f15c6b9925f6fadc21b106161cb8e4a84ff2fd30b318cc5d +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_14-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..050ddf79b1e951fe19b12aa0d68cd44d95b2ad24 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9dd56a5c6cd3987673903beb3a270764f564a0a926634d29737ff109a024e3 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_15-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..023a5132b43ef35d5164e70ede49fd78e76ee6c3 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a301e8989cd699e605248332f23bc6263d689baa26ef4630fc86cae1dee18aa +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_16-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..360b5dc87c7f09c10369f6d3570a6f8297023e83 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05e6a463fdeb817a8cc663ee386a1a7a282bd4afac259aa6916e8268740b8503 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_17-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7befdbb9771cc6d87184d21e090cdd29ef587bb --- /dev/null +++ b/1b11b5100m/global_step2891/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb18fdbd1a197002313a597251dc2c43be08d48687950fb7f9e4943a6ae5879 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_18-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..273302811a187fcf214d729723df9d679f9ebea2 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1610d845d968d9cd75f0970316b394dd1af9ccc40516e00b969299a6f830e0c1 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_19-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b1a27715db4fbd40430180c38d0575b41e0272e --- /dev/null +++ b/1b11b5100m/global_step2891/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d38d917faf85d13b853d1df3c0fede329e859a12ff03dc0ac9b0228f310db935 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_20-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a830acc54a20c45898c32ea3df892b7e8b9eb7d --- /dev/null +++ b/1b11b5100m/global_step2891/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f55c597940ec8c50e5efb318495c7f33ffe163e4f1ece7f35e42cee6ee5ccaf9 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_21-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..455d95ad091709962ceb64a51669f8f8395d7d4f --- /dev/null +++ b/1b11b5100m/global_step2891/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df9f25d5946772b9f807c040d6d947940c31d739d8db77e239812dc9da0788ed +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_22-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fc7b8ce90dc3f28ddd141daa170a43c8191fb70 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30f48371cb3770782d8f9c91eb14a4edb9af9b0972b31618e88557e8e536a231 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_23-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74dd03a1b00e8eafdfeefcc7f4db7b1af6f74787 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd49a45da90a5c1b6f576749a7ba591e6aef97f7a976335475f8748a35da7ca +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_24-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bdeba2cc2ed2215e966f9cb32a0954c8dc8c99c --- /dev/null +++ b/1b11b5100m/global_step2891/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f2fcdc380e65d3ecfab24bfa3b1fd874dc3b14caf4aff5cdef581455a693a37 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_25-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..594ecd73788ae104af5846248cf182cc5ee4e4ee --- /dev/null +++ b/1b11b5100m/global_step2891/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97cf8f3e86f8c26a5852479d2e255fdf33bba24417bc910fe1a718c2600b8528 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_26-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a0549bb319b89283deac2cc982e49f128469a73 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e27f0aa88c7c390051ae2bab5d6c52113f5d50cdab0516da5799d2be8670bdc3 +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_27-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c5c908e847f4b9ed27c55bb81635443d9d0d34a --- /dev/null +++ b/1b11b5100m/global_step2891/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a0697e7f37a170a4e613a64559ca60f2a1feba5ef84c60097ec6ac98dc5875b +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_28-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64bdf572fd7570d791caa8a091276757a66b1219 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a4dc15b6cdca988145178d0eb7944c33fc8a9d18ac45d353cdd475560ef243d +size 77121283 diff --git a/1b11b5100m/global_step2891/layer_30-model_00-model_states.pt b/1b11b5100m/global_step2891/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9deeb17c7c1569c79b044a3c3d5ea1e08660a18 --- /dev/null +++ b/1b11b5100m/global_step2891/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0820755b70225824921cb3281573626f17831b91a1a93d2a9f93b00507a1dee +size 8387 diff --git a/1b11b5100m/global_step2891/mp_rank_00_model_states.pt b/1b11b5100m/global_step2891/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b693ecac31149780cf255f7d8ecc260321fe6ae7 --- /dev/null +++ b/1b11b5100m/global_step2891/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28d699d091d4e5a153c7a69c5f98a6a8f3baa559d8db447b91c73d82efeeaa71 +size 43827 diff --git a/1b11b5100m/logs/3300038.err b/1b11b5100m/logs/3300038.err new file mode 100644 index 0000000000000000000000000000000000000000..18f5a31cc0f09f033ac2f623a0a421a7321c064a --- /dev/null +++ b/1b11b5100m/logs/3300038.err @@ -0,0 +1,1118 @@ +1: 2023-03-13 22:10:48.855295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-03-13 22:10:48.855300: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-03-13 22:10:48.855314: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-03-13 22:10:48.855310: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-03-13 22:10:48.855317: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-03-13 22:10:48.855321: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-03-13 22:10:48.855317: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-03-13 22:10:48.855317: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889469: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889472: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889468: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889479: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889487: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889489: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889483: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-03-13 22:10:48.889492: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925687: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925696: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925690: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925700: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925688: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925702: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925700: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:48.925696: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038747: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038760: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038756: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038764: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038765: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038766: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-03-13 22:10:49.038770: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323569: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323575: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323580: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323581: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323585: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323586: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323587: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-03-13 22:10:49.323575: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361429: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361430: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361431: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361428: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361420: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361435: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361419: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-03-13 22:10:49.361435: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495740: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495749: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495746: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495754: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495741: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495751: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495747: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-03-13 22:10:49.495763: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614797: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614805: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614806: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614813: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614816: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614817: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614818: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-03-13 22:10:49.614821: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-03-13 22:10:50.672828: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.672836: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.672829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.672839: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.672841: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.672846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.672847: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.672835: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:50.673079: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-03-13 22:10:50.673078: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-03-13 22:10:50.673082: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-03-13 22:10:50.673084: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-03-13 22:10:50.673086: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-03-13 22:10:50.673087: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-03-13 22:10:50.673089: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-03-13 22:10:50.673093: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806019: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806016: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806026: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:50.806299: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806301: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806304: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806305: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806306: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806312: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806314: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:50.806315: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.847854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.847850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.847847: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.847856: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.847858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.847864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.847865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.847858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:50.848318: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.848321: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.848326: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.848326: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.848328: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.848329: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.848330: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-03-13 22:10:50.848334: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.952798: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.952806: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.952801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.952808: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.952811: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.952810: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.952812: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.952807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:50.953219: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.953227: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.953229: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.953232: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.953232: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.953235: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.953235: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-03-13 22:10:50.953240: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.167943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.167947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.167942: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.167950: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.167948: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.167947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.167957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.167957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:51.168357: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.168357: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.168362: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.168370: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.168369: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.168370: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.168374: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-03-13 22:10:51.168372: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.226945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.226945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.226955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.226954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.226955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.226954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.226955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.226959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:51.227346: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.227349: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.227352: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.227353: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.227355: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.227359: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.227360: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-03-13 22:10:51.227363: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.381647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.381643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.381655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.381653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.381660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.381663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.381653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.381658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:51.382058: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.382062: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.382067: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.382067: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.382071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.382071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.382074: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-03-13 22:10:51.382078: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428298: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428317: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428310: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428310: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428310: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428313: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:51.428488: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428493: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428494: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428497: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428496: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428499: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428505: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-03-13 22:10:51.428508: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-03-13 22:10:56.620438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.620446: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.620449: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.620444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.620451: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.620448: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.620453: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.620455: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-03-13 22:10:56.622888: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622812: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-03-13 22:10:56.622887: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-03-13 22:10:56.622894: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-03-13 22:10:56.622893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622822: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-03-13 22:10:56.622892: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-03-13 22:10:56.622897: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.622827: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.622905: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-03-13 22:10:56.622905: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-03-13 22:10:56.622910: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-03-13 22:10:56.622911: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-03-13 22:10:56.622912: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-03-13 22:10:56.622911: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-03-13 22:10:56.622954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.622956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-03-13 22:10:56.622969: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-03-13 22:10:56.622970: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624724: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624727: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624727: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624727: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624740: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624743: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624747: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624745: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624746: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624748: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624750: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-03-13 22:10:56.624825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-03-13 22:10:56.624837: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.632503: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.632502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.632512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.632510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.632516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.632511: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.632514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.632517: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634209: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634220: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.634221: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.634223: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.634226: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.634227: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.634228: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.634230: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-03-13 22:10:56.634301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-03-13 22:10:56.634314: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.639500: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.639507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.639510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.639513: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.639513: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.639518: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.639519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.639524: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642164: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642172: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.642174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.642174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.642175: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.642181: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.642181: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.642343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-03-13 22:10:56.642360: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-03-13 22:10:56.642369: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.644062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.644071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.644069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.644073: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.644074: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.644072: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.644076: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.644077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646132: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.646140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.646143: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.646145: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.646146: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.646148: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.646179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-03-13 22:10:56.646192: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-03-13 22:10:56.646193: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.646408: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.646423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.646420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.646421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.646416: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.646418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.646418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.646427: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650134: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650144: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650151: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.650156: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652353: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652356: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652359: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652364: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-03-13 22:10:56.652364: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-03-13 22:10:56.652370: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-03-13 22:10:56.652371: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-03-13 22:10:56.652372: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-03-13 22:10:56.652373: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-03-13 22:10:56.652375: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-03-13 22:10:56.652417: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-03-13 22:10:56.652429: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.652950: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.652945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.652957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.652959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.652956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.652961: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.652967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.652975: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655115: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655127: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.655134: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.655136: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.655136: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.655138: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.655139: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.655163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-03-13 22:10:56.655179: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-03-13 22:10:56.655183: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648059: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648081: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648081: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648081: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648082: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648082: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648085: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-03-13 22:10:56.648141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-03-13 22:10:56.648141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_upper_triang_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +1: Successfully preprocessed all matching files. +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +2: +2: +2: +2: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: +3: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: +6: +6: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +0: Building extension module utils... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module utils... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils...Loading extension module utils...Loading extension module utils...Loading extension module utils... +0: +0: Loading extension module utils... +0: +0: +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +0: +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +0: +0: Loading extension module utils...Loading extension module utils... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils...Loading extension module utils... +4: +4: No modifications detected for re-loaded extension module utils, skipping build step... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +4: +4: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +5: +5: +5: Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: +5: +7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: Loading extension module utils...Loading extension module utils... +7: +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +7: +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +6: +6: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +6: +6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +6: +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/1b11b5100m/logs/3300038.out b/1b11b5100m/logs/3300038.out new file mode 100644 index 0000000000000000000000000000000000000000..aa28f812823ecc157f6dd36f977ea0c8b353425f --- /dev/null +++ b/1b11b5100m/logs/3300038.out @@ -0,0 +1,8491 @@ +Model parameters: d_model 1792 ffw_size 7168 kv_size 128 n_heads 14 n_layers 26 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 26 --hidden-size 1792 --num-attention-heads 14 --kv-channels 128 --ffn-hidden-size 7168 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --loss-scale 12 --clip-grad 1.0 --kill-switch-path kill-switch-1b11b5100mval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --no-load-optim --reset-progress --override-lr-scheduler --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-only true --eval-iters 100 --tensorboard-dir tensorboard_1b11b5100mval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_1b11b5100m --load checkpoints_1b11b5100m --train-weighted-split-paths-path train100m.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3300038.json --zero-stage 0 +START 3300038: Mon 13 Mar 2023 10:09:46 PM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 41.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 47.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 47.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 42.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 41.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 35.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 44.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 41.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 38.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +2: Launching on nid006586 (2/8), master nid006584 port 9999, GPUs 8, CUDA: True +0: Launching on nid006584 (0/8), master nid006584 port 9999, GPUs 8, CUDA: True +1: Launching on nid006585 (1/8), master nid006584 port 9999, GPUs 8, CUDA: True +6: Launching on nid006590 (6/8), master nid006584 port 9999, GPUs 8, CUDA: True +7: Launching on nid006591 (7/8), master nid006584 port 9999, GPUs 8, CUDA: True +5: Launching on nid006589 (5/8), master nid006584 port 9999, GPUs 8, CUDA: True +4: Launching on nid006588 (4/8), master nid006584 port 9999, GPUs 8, CUDA: True +3: Launching on nid006587 (3/8), master nid006584 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/3300038.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... True +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 7168 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1792 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-1b11b5100mval +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_1b11b5100m +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... 12.0 +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 14 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 26 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_1b11b5100m +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_1b11b5100mval +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-03-13 22:11:25,295] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +7: > setting tensorboard ... +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.110 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 87 +0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.o scaled_upper_triang_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 63 +0: [1/1] c++ scaled_masked_softmax_hip.o scaled_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: [1/1] c++ layer_norm_hip_kernel.cuda.o layer_norm_cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so +0: >>> done with compiling and loading fused kernels. Compilation time: 18.224 seconds +0: time to initialize megatron (seconds): 2.414 +0: [after megatron is initialized] datetime: 2023-03-13 22:11:46 +0: building GPT model ... +0: [2023-03-13 22:11:46,545] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-03-13 22:11:46,546] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-03-13 22:11:46,546] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.71 GB, percent = 6.1% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-03-13 22:11:48,591] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=33 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: ParallelTransformerLayerPipe +0: 23: ParallelTransformerLayerPipe +0: 24: ParallelTransformerLayerPipe +0: 25: ParallelTransformerLayerPipe +0: 26: ParallelTransformerLayerPipe +0: 27: ParallelTransformerLayerPipe +0: 28: ParallelTransformerLayerPipe +0: 29: undo +0: 30: MixedFusedLayerNorm +0: 31: EmbeddingPipe +0: 32: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-03-13 22:11:48,823] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-03-13 22:11:48,824] [INFO] [utils.py:828:see_memory_usage] MA 2.05 GB Max_MA 2.05 GB CA 2.19 GB Max_CA 2 GB +0: [2023-03-13 22:11:48,824] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-03-13 22:11:48,826] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-03-13 22:12:02,142] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-03-13 22:12:02,143] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-03-13 22:12:02,143] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-03-13 22:12:02,155] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-03-13 22:12:02,155] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-03-13 22:12:02,274] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-03-13 22:12:02,275] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB Max_MA 2.06 GB CA 2.19 GB Max_CA 2 GB +0: [2023-03-13 22:12:02,275] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.46 GB, percent = 6.2% +0: ninja: no work to do. +0: Time to load utils op: 0.1517930030822754 seconds +0: Time to load utils op: 0.10261893272399902 seconds +0: Time to load utils op: 0.0005640983581542969 seconds +0: Time to load utils op: 0.2053520679473877 secondsTime to load utils op: 0.20504403114318848 secondsTime to load utils op: 0.20524120330810547 seconds +0: +0: +0: Time to load utils op: 0.204941987991333 secondsTime to load utils op: 0.20469236373901367 seconds +0: Time to load utils op: 0.20519208908081055 seconds +0: +1: Time to load utils op: 0.2114849090576172 seconds +1: Time to load utils op: 0.2114114761352539 seconds +1: Time to load utils op: 0.21193909645080566 seconds +1: Time to load utils op: 0.2114253044128418 seconds +1: Time to load utils op: 0.21140766143798828 secondsTime to load utils op: 0.21153903007507324 secondsTime to load utils op: 0.2113938331604004 seconds +1: +1: +1: Time to load utils op: 0.21206927299499512 seconds +3: Time to load utils op: 0.21054983139038086 seconds +3: Time to load utils op: 0.21056652069091797 secondsTime to load utils op: 0.21056437492370605 seconds +3: +3: Time to load utils op: 0.2105700969696045 secondsTime to load utils op: 0.21058440208435059 seconds +3: +3: Time to load utils op: 0.21057438850402832 secondsTime to load utils op: 0.21058106422424316 secondsTime to load utils op: 0.21058106422424316 seconds +3: +3: +2: Time to load utils op: 0.21229338645935059 seconds +2: Time to load utils op: 0.21230053901672363 seconds +2: Time to load utils op: 0.21230030059814453 seconds +2: Time to load utils op: 0.21231698989868164 secondsTime to load utils op: 0.21231436729431152 seconds +2: +2: Time to load utils op: 0.2123250961303711 secondsTime to load utils op: 0.2123422622680664 seconds +2: Time to load utils op: 0.21233820915222168 seconds +2: +4: Time to load utils op: 0.21115422248840332 seconds +4: Time to load utils op: 0.2111954689025879 seconds +4: Time to load utils op: 0.2111492156982422 seconds +4: Time to load utils op: 0.211226224899292 seconds +4: Time to load utils op: 0.21123623847961426 secondsTime to load utils op: 0.211228609085083 seconds +4: Time to load utils op: 0.21123290061950684 seconds +4: +4: Time to load utils op: 0.2112438678741455 seconds +5: Time to load utils op: 0.21201276779174805 seconds +5: Time to load utils op: 0.21219158172607422 seconds +5: Time to load utils op: 0.21222209930419922 seconds +5: Time to load utils op: 0.21223235130310059 seconds +5: Time to load utils op: 0.21224331855773926 seconds +5: Time to load utils op: 0.21224737167358398 secondsTime to load utils op: 0.2122492790222168 seconds +5: Time to load utils op: 0.21225476264953613 seconds +5: +7: Time to load utils op: 0.21164965629577637 seconds +7: Time to load utils op: 0.21165227890014648 seconds +6: Time to load utils op: 0.21283435821533203 secondsTime to load utils op: 0.21282529830932617 seconds +6: +6: Time to load utils op: 0.21284198760986328 secondsTime to load utils op: 0.2128448486328125 seconds +6: +7: Time to load utils op: 0.211669921875 seconds +6: Time to load utils op: 0.21285486221313477 secondsTime to load utils op: 0.2128453254699707 seconds +6: +6: Time to load utils op: 0.21284842491149902 secondsTime to load utils op: 0.21285176277160645 seconds +6: +7: Time to load utils op: 0.21159934997558594 seconds +7: Time to load utils op: 0.21170330047607422 seconds +7: Time to load utils op: 0.21170449256896973 seconds +7: Time to load utils op: 0.2117147445678711 seconds +7: Time to load utils op: 0.2117326259613037 seconds +0: Time to load utils op: 0.0004329681396484375 seconds +0: Time to load utils op: 0.0003991127014160156 seconds +0: Time to load utils op: 0.0004088878631591797 seconds +0: Time to load utils op: 0.00043892860412597656 seconds +0: Time to load utils op: 0.00045680999755859375 seconds +0: Time to load utils op: 0.0004012584686279297 seconds +1: Time to load utils op: 0.001009225845336914 seconds +4: Time to load utils op: 0.0009446144104003906 seconds +3: Time to load utils op: 0.0010082721710205078 seconds +4: Time to load utils op: 0.0008869171142578125 seconds +1: Time to load utils op: 0.0011601448059082031 seconds +4: Time to load utils op: 0.0010771751403808594 secondsTime to load utils op: 0.0010128021240234375 seconds +4: +4: Time to load utils op: 0.0012214183807373047 seconds +2: Time to load utils op: 0.00135040283203125 seconds +4: Time to load utils op: 0.0011408329010009766 seconds +4: Time to load utils op: 0.0011990070343017578 seconds +4: Time to load utils op: 0.0013060569763183594 seconds +3: Time to load utils op: 0.0011687278747558594 secondsTime to load utils op: 0.0011322498321533203 seconds +3: +1: Time to load utils op: 0.0011582374572753906 seconds +3: Time to load utils op: 0.0011746883392333984 seconds +3: Time to load utils op: 0.0011882781982421875 seconds +1: Time to load utils op: 0.0012664794921875 seconds +1: Time to load utils op: 0.001287698745727539 seconds +1: Time to load utils op: 0.0013628005981445312 secondsTime to load utils op: 0.0013439655303955078 seconds +1: +3: Time to load utils op: 0.0012462139129638672 seconds +1: Time to load utils op: 0.0013899803161621094 seconds +3: Time to load utils op: 0.0013034343719482422 seconds +3: Time to load utils op: 0.0012409687042236328 seconds +2: Time to load utils op: 0.0016906261444091797 seconds +2: Time to load utils op: 0.0017786026000976562 seconds +2: Time to load utils op: 0.0017554759979248047 seconds +2: Time to load utils op: 0.0017805099487304688 secondsTime to load utils op: 0.00177764892578125 seconds +2: +2: Time to load utils op: 0.0018031597137451172 seconds +2: Time to load utils op: 0.0017387866973876953 seconds +7: Time to load utils op: 0.0013737678527832031 seconds +7: Time to load utils op: 0.0013606548309326172 seconds +5: Time to load utils op: 0.0005431175231933594 seconds +5: Time to load utils op: 0.0005323886871337891 seconds +5: Time to load utils op: 0.0005586147308349609 seconds +5: Time to load utils op: 0.0008695125579833984 seconds +5: Time to load utils op: 0.0009791851043701172 seconds +5: Time to load utils op: 0.0009698867797851562 seconds +5: Time to load utils op: 0.0006814002990722656 seconds +7: Time to load utils op: 0.0014731884002685547 secondsTime to load utils op: 0.0014791488647460938 seconds +7: +7: Time to load utils op: 0.0014526844024658203 seconds +7: Time to load utils op: 0.0014879703521728516 seconds +7: Time to load utils op: 0.0013856887817382812 seconds +5: Time to load utils op: 0.000934600830078125 seconds +7: Time to load utils op: 0.0014307498931884766 seconds +6: Time to load utils op: 0.0008180141448974609 secondsTime to load utils op: 0.0006816387176513672 seconds +6: +6: Time to load utils op: 0.000457763671875 seconds +6: Time to load utils op: 0.0007262229919433594 seconds +6: Time to load utils op: 0.0004329681396484375 secondsTime to load utils op: 0.0005748271942138672 seconds +6: +6: Time to load utils op: 0.0005598068237304688 seconds +6: Time to load utils op: 0.0004937648773193359 seconds +0: [2023-03-13 22:12:02,499] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-03-13 22:12:02,499] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB Max_MA 2.04 GB CA 2.19 GB Max_CA 2 GB +0: [2023-03-13 22:12:02,499] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.5 GB, percent = 6.3% +0: [2023-03-13 22:12:02,628] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-03-13 22:12:02,628] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB Max_MA 4.24 GB CA 5.44 GB Max_CA 5 GB +0: [2023-03-13 22:12:02,629] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:02,732] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-03-13 22:12:02,733] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB Max_MA 4.24 GB CA 5.44 GB Max_CA 5 GB +0: [2023-03-13 22:12:02,733] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:02,836] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-03-13 22:12:02,837] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-03-13 22:12:02,837] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:02,937] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-03-13 22:12:02,938] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-03-13 22:12:02,938] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:03,042] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-03-13 22:12:03,043] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-03-13 22:12:03,043] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:03,143] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-03-13 22:12:03,143] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-03-13 22:12:03,144] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:03,250] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-03-13 22:12:03,251] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB Max_MA 6.32 GB CA 8.34 GB Max_CA 8 GB +0: [2023-03-13 22:12:03,251] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:03,352] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-03-13 22:12:03,353] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB Max_MA 6.32 GB CA 8.34 GB Max_CA 8 GB +0: [2023-03-13 22:12:03,353] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.61 GB, percent = 6.3% +0: [2023-03-13 22:12:03,353] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-03-13 22:12:03,353] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-03-13 22:12:03,353] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-03-13 22:12:03,353] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-03-13 22:12:03,354] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-03-13 22:12:03,355] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-03-13 22:12:03,356] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-03-13 22:12:03,356] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-03-13 22:12:03,356] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-03-13 22:12:03,356] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-03-13 22:12:03,356] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-03-13 22:12:03,356] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-03-13 22:12:03,356] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.00040411949157714844 seconds +0: [2023-03-13 22:12:03,356] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-03-13 22:12:03,410] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=33 [0, 33) STAGE_PARAMS=1096338432 (1096.338M) TOTAL_PARAMS=1096338432 (1096.338M) UNIQUE_PARAMS=1096338432 (1096.338M) +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +6: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +0: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +0: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +3: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +1: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt... +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +6: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +3: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/mp_rank_00_model_states.pt. +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-03-13 22:12:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-03-13 22:12:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-03-13 22:12:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-03-13 22:12:03,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-03-13 22:12:03,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-03-13 22:12:03,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:03,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:03,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:03,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:03,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:03,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-03-13 22:12:03,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:03,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-03-13 22:12:03,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-03-13 22:12:03,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:03,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:03,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-03-13 22:12:03,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:03,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:03,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-03-13 22:12:03,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-03-13 22:12:03,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-03-13 22:12:03,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:03,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:03,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:03,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:03,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-03-13 22:12:03,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:03,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:03,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:03,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:03,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-03-13 22:12:04,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-03-13 22:12:04,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:04,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-03-13 22:12:04,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-03-13 22:12:04,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:04,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:04,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-03-13 22:12:04,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-03-13 22:12:04,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-03-13 22:12:04,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:04,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:04,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-03-13 22:12:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:04,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:04,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:04,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-03-13 22:12:04,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-03-13 22:12:04,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-03-13 22:12:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-03-13 22:12:04,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-03-13 22:12:04,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-03-13 22:12:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-03-13 22:12:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-03-13 22:12:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-03-13 22:12:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-03-13 22:12:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-03-13 22:12:04,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-03-13 22:12:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-03-13 22:12:04,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-03-13 22:12:04,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-03-13 22:12:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-03-13 22:12:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-03-13 22:12:04,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-03-13 22:12:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-03-13 22:12:04,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:04,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-03-13 22:12:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:04,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:04,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:04,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-03-13 22:12:04,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:04,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-03-13 22:12:04,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-03-13 22:12:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-03-13 22:12:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-03-13 22:12:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:04,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:04,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:04,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:04,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:04,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-03-13 22:12:04,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-03-13 22:12:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:04,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-03-13 22:12:04,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-03-13 22:12:04,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:04,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-03-13 22:12:05,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-03-13 22:12:05,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:05,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-03-13 22:12:05,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:05,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-03-13 22:12:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-03-13 22:12:05,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-03-13 22:12:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-03-13 22:12:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-03-13 22:12:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-03-13 22:12:05,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-03-13 22:12:05,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-03-13 22:12:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-03-13 22:12:05,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-03-13 22:12:05,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-03-13 22:12:05,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-03-13 22:12:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-03-13 22:12:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-03-13 22:12:05,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-03-13 22:12:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-03-13 22:12:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-03-13 22:12:05,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-03-13 22:12:05,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-03-13 22:12:05,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-03-13 22:12:05,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-03-13 22:12:05,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-03-13 22:12:05,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-03-13 22:12:05,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-03-13 22:12:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-03-13 22:12:05,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-03-13 22:12:05,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-03-13 22:12:05,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-03-13 22:12:05,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-03-13 22:12:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-03-13 22:12:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-03-13 22:12:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-03-13 22:12:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-03-13 22:12:05,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-03-13 22:12:05,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-03-13 22:12:05,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-03-13 22:12:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-03-13 22:12:05,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-03-13 22:12:05,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-03-13 22:12:05,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-03-13 22:12:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-03-13 22:12:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-03-13 22:12:05,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-03-13 22:12:05,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-03-13 22:12:05,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-03-13 22:12:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-03-13 22:12:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-03-13 22:12:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-03-13 22:12:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:05,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-03-13 22:12:05,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-03-13 22:12:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:05,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-03-13 22:12:05,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-03-13 22:12:05,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-03-13 22:12:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:05,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-03-13 22:12:06,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-03-13 22:12:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:06,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:06,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-03-13 22:12:06,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-03-13 22:12:06,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:06,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:06,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:06,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-03-13 22:12:06,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-03-13 22:12:06,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:06,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:06,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-03-13 22:12:06,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-03-13 22:12:06,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-03-13 22:12:06,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-03-13 22:12:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-03-13 22:12:06,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-03-13 22:12:06,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-03-13 22:12:06,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-03-13 22:12:06,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-03-13 22:12:06,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-03-13 22:12:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-03-13 22:12:06,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-03-13 22:12:06,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-03-13 22:12:06,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-03-13 22:12:06,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-03-13 22:12:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-03-13 22:12:06,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-03-13 22:12:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-03-13 22:12:06,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-03-13 22:12:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-03-13 22:12:06,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-03-13 22:12:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-03-13 22:12:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-03-13 22:12:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-03-13 22:12:06,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:06,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-03-13 22:12:06,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-03-13 22:12:06,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-03-13 22:12:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-03-13 22:12:06,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:06,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:06,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:06,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-03-13 22:12:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:06,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:06,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:06,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:06,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-03-13 22:12:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-03-13 22:12:06,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-03-13 22:12:06,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-03-13 22:12:06,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-03-13 22:12:06,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-03-13 22:12:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-03-13 22:12:06,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:06,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:06,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:06,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:06,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:06,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:07,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:07,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-03-13 22:12:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-03-13 22:12:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-03-13 22:12:07,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-03-13 22:12:07,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-03-13 22:12:07,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-03-13 22:12:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-03-13 22:12:07,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-03-13 22:12:07,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-03-13 22:12:07,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-03-13 22:12:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-03-13 22:12:07,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-03-13 22:12:07,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-03-13 22:12:07,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-03-13 22:12:07,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-03-13 22:12:07,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-03-13 22:12:07,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-03-13 22:12:07,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-03-13 22:12:07,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-03-13 22:12:07,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-03-13 22:12:07,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-03-13 22:12:07,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-03-13 22:12:07,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-03-13 22:12:07,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-03-13 22:12:07,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-03-13 22:12:07,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-03-13 22:12:07,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-03-13 22:12:07,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-03-13 22:12:07,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-03-13 22:12:07,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-03-13 22:12:07,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-03-13 22:12:07,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-03-13 22:12:07,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-03-13 22:12:07,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-03-13 22:12:07,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-03-13 22:12:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-03-13 22:12:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-03-13 22:12:07,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-03-13 22:12:07,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-03-13 22:12:07,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-03-13 22:12:07,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-03-13 22:12:07,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-03-13 22:12:07,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-03-13 22:12:07,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-03-13 22:12:07,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-03-13 22:12:07,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-03-13 22:12:07,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-03-13 22:12:07,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-03-13 22:12:07,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-03-13 22:12:07,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-03-13 22:12:07,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-03-13 22:12:07,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:07,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:07,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-03-13 22:12:07,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:07,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:07,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:07,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:07,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:07,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-03-13 22:12:07,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:07,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-03-13 22:12:07,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-03-13 22:12:07,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-03-13 22:12:07,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-03-13 22:12:07,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:07,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-03-13 22:12:07,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:07,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:07,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:07,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:07,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-03-13 22:12:08,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-03-13 22:12:08,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-03-13 22:12:08,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-03-13 22:12:08,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-03-13 22:12:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-03-13 22:12:08,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-03-13 22:12:08,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-03-13 22:12:08,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-03-13 22:12:08,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-03-13 22:12:08,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-03-13 22:12:08,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-03-13 22:12:08,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-03-13 22:12:08,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-03-13 22:12:08,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-03-13 22:12:08,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-03-13 22:12:08,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-03-13 22:12:08,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-03-13 22:12:08,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-03-13 22:12:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-03-13 22:12:08,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-03-13 22:12:08,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-03-13 22:12:08,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-03-13 22:12:08,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-03-13 22:12:08,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-03-13 22:12:08,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-03-13 22:12:08,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-03-13 22:12:08,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-03-13 22:12:08,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-03-13 22:12:08,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-03-13 22:12:08,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-03-13 22:12:08,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-03-13 22:12:08,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-03-13 22:12:08,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-03-13 22:12:08,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-03-13 22:12:08,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:08,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:08,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:08,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-03-13 22:12:08,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:08,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:08,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-03-13 22:12:08,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-03-13 22:12:08,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-03-13 22:12:08,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:08,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-03-13 22:12:08,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:08,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-03-13 22:12:08,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:08,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:08,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:08,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:08,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:08,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:08,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:08,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-03-13 22:12:08,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-03-13 22:12:08,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:08,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:08,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:08,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:09,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:09,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:09,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-03-13 22:12:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-03-13 22:12:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-03-13 22:12:09,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:09,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:09,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-03-13 22:12:09,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:09,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-03-13 22:12:09,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-03-13 22:12:09,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-03-13 22:12:09,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-03-13 22:12:09,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-03-13 22:12:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-03-13 22:12:09,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-03-13 22:12:09,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-03-13 22:12:09,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-03-13 22:12:09,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-03-13 22:12:09,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-03-13 22:12:09,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-03-13 22:12:09,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-03-13 22:12:09,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-03-13 22:12:09,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-03-13 22:12:09,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-03-13 22:12:09,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-03-13 22:12:09,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-03-13 22:12:09,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-03-13 22:12:09,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-03-13 22:12:09,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-03-13 22:12:09,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-03-13 22:12:09,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-03-13 22:12:09,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-03-13 22:12:09,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-03-13 22:12:09,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-03-13 22:12:09,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-03-13 22:12:09,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-03-13 22:12:09,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-03-13 22:12:09,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-03-13 22:12:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-03-13 22:12:09,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-03-13 22:12:09,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-03-13 22:12:09,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-03-13 22:12:09,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-03-13 22:12:09,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-03-13 22:12:09,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-03-13 22:12:09,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:09,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:09,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:09,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:09,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:09,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:09,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:09,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-03-13 22:12:09,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-03-13 22:12:09,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-03-13 22:12:09,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-03-13 22:12:09,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-03-13 22:12:09,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:09,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-03-13 22:12:09,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:09,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:09,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:09,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:09,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:09,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-03-13 22:12:09,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:09,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-03-13 22:12:09,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:09,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:09,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:09,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:09,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:09,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:09,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-03-13 22:12:09,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:09,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:09,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:09,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:09,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-03-13 22:12:09,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-03-13 22:12:09,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:09,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-03-13 22:12:09,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:09,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:09,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:09,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-03-13 22:12:09,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:09,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-03-13 22:12:09,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:09,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-03-13 22:12:09,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:09,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-03-13 22:12:10,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-03-13 22:12:10,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-03-13 22:12:10,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:10,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:10,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:10,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:10,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:10,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:10,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-03-13 22:12:10,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-03-13 22:12:10,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-03-13 22:12:10,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-03-13 22:12:10,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-03-13 22:12:10,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-03-13 22:12:10,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-03-13 22:12:10,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-03-13 22:12:10,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-03-13 22:12:10,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-03-13 22:12:10,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-03-13 22:12:10,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-03-13 22:12:10,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-03-13 22:12:10,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-03-13 22:12:10,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-03-13 22:12:10,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-03-13 22:12:10,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-03-13 22:12:10,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-03-13 22:12:10,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-03-13 22:12:10,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-03-13 22:12:10,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-03-13 22:12:10,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-03-13 22:12:10,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-03-13 22:12:10,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-03-13 22:12:10,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-03-13 22:12:10,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-03-13 22:12:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-03-13 22:12:10,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-03-13 22:12:10,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-03-13 22:12:10,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-03-13 22:12:10,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-03-13 22:12:10,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-03-13 22:12:10,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-03-13 22:12:10,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-03-13 22:12:10,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-03-13 22:12:10,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-03-13 22:12:10,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-03-13 22:12:10,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-03-13 22:12:10,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-03-13 22:12:10,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:10,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:10,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:10,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:10,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-03-13 22:12:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:10,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:10,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-03-13 22:12:10,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-03-13 22:12:10,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:10,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-03-13 22:12:10,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-03-13 22:12:10,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:10,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:10,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-03-13 22:12:10,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-03-13 22:12:10,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:10,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-03-13 22:12:10,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-03-13 22:12:10,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-03-13 22:12:10,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-03-13 22:12:10,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:10,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:10,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-03-13 22:12:10,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-03-13 22:12:10,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-03-13 22:12:10,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:10,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:10,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:10,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-03-13 22:12:10,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:10,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-03-13 22:12:10,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:10,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-03-13 22:12:11,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:11,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-03-13 22:12:11,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:11,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-03-13 22:12:11,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-03-13 22:12:11,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-03-13 22:12:11,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +1: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-03-13 22:12:11,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-03-13 22:12:11,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:11,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-03-13 22:12:11,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-03-13 22:12:11,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +4: [2023-03-13 22:12:11,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +4: [2023-03-13 22:12:11,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +4: [2023-03-13 22:12:11,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-03-13 22:12:11,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-03-13 22:12:11,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-03-13 22:12:11,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +7: [2023-03-13 22:12:11,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-03-13 22:12:11,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +0: [2023-03-13 22:12:11,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-03-13 22:12:11,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-03-13 22:12:11,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-03-13 22:12:11,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-03-13 22:12:11,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +1: [2023-03-13 22:12:11,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:11,833] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +1: [2023-03-13 22:12:11,839] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +7: [2023-03-13 22:12:11,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:11,882] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +5: [2023-03-13 22:12:11,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:11,887] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +7: [2023-03-13 22:12:11,889] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +5: [2023-03-13 22:12:11,894] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +1: [2023-03-13 22:12:11,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:11,897] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +1: [2023-03-13 22:12:11,903] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +6: [2023-03-13 22:12:11,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:11,918] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +2: [2023-03-13 22:12:11,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-03-13 22:12:11,924] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +6: [2023-03-13 22:12:11,925] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +2: [2023-03-13 22:12:11,931] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +1: [2023-03-13 22:12:11,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:11,986] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +1: [2023-03-13 22:12:11,993] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +4: [2023-03-13 22:12:11,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-03-13 22:12:11,999] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +7: [2023-03-13 22:12:12,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:12,003] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +4: [2023-03-13 22:12:12,006] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +5: [2023-03-13 22:12:12,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:12,009] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +7: [2023-03-13 22:12:12,009] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +5: [2023-03-13 22:12:12,015] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +3: [2023-03-13 22:12:12,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,019] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +4: [2023-03-13 22:12:12,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-03-13 22:12:12,020] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +0: [2023-03-13 22:12:12,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,022] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +3: [2023-03-13 22:12:12,026] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +4: [2023-03-13 22:12:12,026] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +0: [2023-03-13 22:12:12,028] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +2: [2023-03-13 22:12:12,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +2: [2023-03-13 22:12:12,043] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +4: [2023-03-13 22:12:12,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-03-13 22:12:12,048] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +2: [2023-03-13 22:12:12,049] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +4: [2023-03-13 22:12:12,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-03-13 22:12:12,054] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +4: [2023-03-13 22:12:12,055] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +1: [2023-03-13 22:12:12,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:12,058] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +1: [2023-03-13 22:12:12,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:12,059] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +7: [2023-03-13 22:12:12,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:12,062] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +4: [2023-03-13 22:12:12,063] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +3: [2023-03-13 22:12:12,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:12,064] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +3: [2023-03-13 22:12:12,064] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +7: [2023-03-13 22:12:12,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:12,065] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +1: [2023-03-13 22:12:12,067] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +5: [2023-03-13 22:12:12,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:12,066] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +5: [2023-03-13 22:12:12,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:12,067] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +6: [2023-03-13 22:12:12,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:12,069] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +7: [2023-03-13 22:12:12,069] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +3: [2023-03-13 22:12:12,070] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +0: [2023-03-13 22:12:12,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,072] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +7: [2023-03-13 22:12:12,072] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +5: [2023-03-13 22:12:12,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:12,073] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +5: [2023-03-13 22:12:12,073] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +5: [2023-03-13 22:12:12,074] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +6: [2023-03-13 22:12:12,075] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +2: [2023-03-13 22:12:12,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-03-13 22:12:12,076] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +0: [2023-03-13 22:12:12,078] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +5: [2023-03-13 22:12:12,079] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +4: [2023-03-13 22:12:12,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-03-13 22:12:12,080] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +0: [2023-03-13 22:12:12,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,081] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +2: [2023-03-13 22:12:12,083] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +3: [2023-03-13 22:12:12,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,084] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +4: [2023-03-13 22:12:12,087] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +0: [2023-03-13 22:12:12,087] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +2: [2023-03-13 22:12:12,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-03-13 22:12:12,090] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +3: [2023-03-13 22:12:12,090] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +2: [2023-03-13 22:12:12,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +2: [2023-03-13 22:12:12,093] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +5: [2023-03-13 22:12:12,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:12,093] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +3: [2023-03-13 22:12:12,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,094] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +2: [2023-03-13 22:12:12,097] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +7: [2023-03-13 22:12:12,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:12,099] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +5: [2023-03-13 22:12:12,099] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +2: [2023-03-13 22:12:12,100] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +3: [2023-03-13 22:12:12,101] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +1: [2023-03-13 22:12:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:12,106] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +0: [2023-03-13 22:12:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,107] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +0: [2023-03-13 22:12:12,107] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +7: [2023-03-13 22:12:12,107] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +6: [2023-03-13 22:12:12,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:12,108] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +3: [2023-03-13 22:12:12,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,110] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +7: [2023-03-13 22:12:12,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:12,110] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +7: [2023-03-13 22:12:12,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:12,113] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +6: [2023-03-13 22:12:12,114] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +1: [2023-03-13 22:12:12,114] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +0: [2023-03-13 22:12:12,115] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +0: [2023-03-13 22:12:12,115] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +3: [2023-03-13 22:12:12,117] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +4: [2023-03-13 22:12:12,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-03-13 22:12:12,117] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +7: [2023-03-13 22:12:12,118] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +7: [2023-03-13 22:12:12,119] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +6: [2023-03-13 22:12:12,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:12,124] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +4: [2023-03-13 22:12:12,124] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +3: [2023-03-13 22:12:12,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,125] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +4: [2023-03-13 22:12:12,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +4: [2023-03-13 22:12:12,129] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +0: [2023-03-13 22:12:12,129] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +6: [2023-03-13 22:12:12,131] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +5: [2023-03-13 22:12:12,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:12,132] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +3: [2023-03-13 22:12:12,132] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +0: [2023-03-13 22:12:12,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,133] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +2: [2023-03-13 22:12:12,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-03-13 22:12:12,136] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +7: [2023-03-13 22:12:12,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,136] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +2: [2023-03-13 22:12:12,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +7: [2023-03-13 22:12:12,136] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +5: [2023-03-13 22:12:12,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-03-13 22:12:12,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +2: [2023-03-13 22:12:12,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +4: [2023-03-13 22:12:12,138] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +5: [2023-03-13 22:12:12,139] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +0: [2023-03-13 22:12:12,140] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +7: [2023-03-13 22:12:12,143] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +5: [2023-03-13 22:12:12,144] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +2: [2023-03-13 22:12:12,144] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +2: [2023-03-13 22:12:12,145] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +3: [2023-03-13 22:12:12,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +6: [2023-03-13 22:12:12,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:12,152] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +6: [2023-03-13 22:12:12,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:12,152] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +3: [2023-03-13 22:12:12,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,154] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +3: [2023-03-13 22:12:12,156] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +6: [2023-03-13 22:12:12,159] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +6: [2023-03-13 22:12:12,159] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +2: [2023-03-13 22:12:12,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-03-13 22:12:12,160] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +4: [2023-03-13 22:12:12,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +3: [2023-03-13 22:12:12,161] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +4: [2023-03-13 22:12:12,161] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +4: [2023-03-13 22:12:12,168] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +6: [2023-03-13 22:12:12,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:12,168] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +2: [2023-03-13 22:12:12,169] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +0: [2023-03-13 22:12:12,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-03-13 22:12:12,171] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +6: [2023-03-13 22:12:12,175] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +0: [2023-03-13 22:12:12,178] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +1: [2023-03-13 22:12:12,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:12,178] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +1: [2023-03-13 22:12:12,186] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +6: [2023-03-13 22:12:12,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-03-13 22:12:12,190] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +6: [2023-03-13 22:12:12,197] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +1: [2023-03-13 22:12:12,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b5100m/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-03-13 22:12:12,226] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +1: [2023-03-13 22:12:12,235] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +0: successfully loaded checkpoint from checkpoints_1b11b5100m at iteration 0 +7: time (ms) | load-checkpoint: 8840.56 +0: estimated model parameters: 1.096338432 +0: estimated model parameters without embeddings: 1.002523648 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-03-13 22:12:12 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.007363 seconds +0: number of documents: 208931 +0: > dataset split: +0: train: +0: document indices in [0, 208931) total of 208931 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.073 seconds +0: total number of samples: 48805 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.027483 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.042 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-03-13 22:12:26 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 26017.34 | train/valid/test-data-iterators-setup: 12663.52 +0: [after training is done] datetime: 2023-03-13 22:12:26 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 3.819042E+00 | lm loss PPL: 4.556054E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 3300038: Mon 13 Mar 2023 10:13:16 PM EET diff --git a/1b11b5100m/sbatch_1b11b5100m.sh b/1b11b5100m/sbatch_1b11b5100m.sh new file mode 100644 index 0000000000000000000000000000000000000000..7572f8e78d1662c18a750e89c4bf4e49f5a9f80f --- /dev/null +++ b/1b11b5100m/sbatch_1b11b5100m.sh @@ -0,0 +1,166 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=1b11b5100m + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train100m.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_1143M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 1516071000 +# -> Samples: 740269 +TRAIN_SAMPLES=740_269 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 7403 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --loss-scale 12 \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/1b11b5100m/sbatch_1b11b5100mval.sh b/1b11b5100m/sbatch_1b11b5100mval.sh new file mode 100644 index 0000000000000000000000000000000000000000..e05d280068c897980270e5d87941a8879c8d690d --- /dev/null +++ b/1b11b5100m/sbatch_1b11b5100mval.sh @@ -0,0 +1,171 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 12:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=1b11b5100mval +VARIANT_CKPT=1b11b5100m + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train100m.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_1143M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 1516071000 +# -> Samples: 740269 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --no-load-optim \ + --reset-progress \ + --override-lr-scheduler \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --loss-scale 12 \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-only true \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/1b11b5100m/tensorboard_1b11b5100m/events.out.tfevents.1677567472.nid007034.106557.0 b/1b11b5100m/tensorboard_1b11b5100m/events.out.tfevents.1677567472.nid007034.106557.0 new file mode 100644 index 0000000000000000000000000000000000000000..e971cb1c7fb2fdececad756e633bc8fa18d9df0c --- /dev/null +++ b/1b11b5100m/tensorboard_1b11b5100m/events.out.tfevents.1677567472.nid007034.106557.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64375e4f1439d00e14e4794e2c597d14e486856d363f82e849774c91f38aa1cb +size 5153659 diff --git a/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1677573182.nid006613.112511.0 b/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1677573182.nid006613.112511.0 new file mode 100644 index 0000000000000000000000000000000000000000..0d88fddcc1cafa7c2f62898fa9b89811b200c9e9 --- /dev/null +++ b/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1677573182.nid006613.112511.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b6c726ddeebf7e95ffa512418211913e5265154f12d62e144e94e8d16b0b16 +size 980 diff --git a/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1678726124.nid005461.89351.0 b/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1678726124.nid005461.89351.0 new file mode 100644 index 0000000000000000000000000000000000000000..1900218dd048873dc26988f8731bd526e0e84ff7 --- /dev/null +++ b/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1678726124.nid005461.89351.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5477f9409ec77de8c306e7ab9014d7371ac68ca97ce486e266d3e6810dd208c5 +size 980 diff --git a/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1678738285.nid006591.109111.0 b/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1678738285.nid006591.109111.0 new file mode 100644 index 0000000000000000000000000000000000000000..2e8d3376e8656e7d8183d66b8bece58d1cc665fd --- /dev/null +++ b/1b11b5100m/tensorboard_1b11b5100mval/events.out.tfevents.1678738285.nid006591.109111.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12fe2121b8ffea0d045fea7b8b7ab758a622e15b270d7ca37a6beec94a27d872 +size 980 diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..74bd1fb02c68bdaf17fd9da589bcdb7cc15392ae --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.22551871811397353, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02136035868888881}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05258955980824922, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001111043010483631}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.24440401749751364, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004069008614986987}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08179555310979232, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015855832653346308}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.023193335272543247, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006829041826176187}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.11105971904247877, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0028341010971909687}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03650350909082802, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010163354215107154}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.051210428458407814, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010759000779014146}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.23873249693202636, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004010114948451509}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.0797122178196054, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015418999962278706}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.050443394209883134, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0010665847609739637}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.23354882611756766, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0038288448419567423}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07835236496419286, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015149830080257896}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b9ddced3ef36c56eeba11a9cd425a7bebbf44c99 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.10303247472010832, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.010542121107444794}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.041949850500467546, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001009545354318454}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.21524248765770787, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003637995924121415}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.06582659671546148, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014010035101277287}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.016602402097909204, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005784273046205441}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.0786634484231439, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002376415534094953}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.025933814076285186, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000854986393522763}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.041053525063470375, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0009905003168813127}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.21058233480126393, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0035581210952122444}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.0644436752805007, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001375438208183654}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.040395747717907556, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0009891799118591429}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.20549607203927084, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0034995349438813944}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.06327032653279559, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013696218498744507}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d7451e5e4c2b5a7ff84689a3ddce9ff01c352782 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.10044377394916508, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.011004903600055578}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.04071859925486269, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.000989754132711725}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2071041537148272, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0036456928364795613}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.06380975510656396, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0013921200872663029}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.016062068279265852, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005807608561522497}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07481502747530522, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002374523180072443}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.024990803032608512, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008536543867796345}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.03972537172680168, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0009640633401932609}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.20198143435079377, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0035579835868813966}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.06228298370655369, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013594322207689349}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.039139482148406846, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0009678826987632891}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.19661295375267737, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0034609252034809553}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.06120828937572442, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013588698800721009}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ef4c2644e5f01652d392c8f28b482e2e29340d35 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.11695012147806128, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.009564035174544912}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.040613137406636475, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.000993472498566677}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.20769491305475654, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003694525077541037}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.06398897669967853, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014297013856788472}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.016119706774984935, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005886522490528559}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07411626939385718, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022817865110913976}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.0250870194649846, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008636090708303351}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.03934321475895627, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0009617791254833742}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2011219485070139, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0035514042056288375}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.06200730116971786, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013847498199754179}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.03873209326674636, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0009654074102020647}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.196124698805037, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0034935507768228203}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.060903726984725515, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001384286630928201}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..15818c633503cdaf9dc34be158f24cee9207b885 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.09979909326915241, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.015027652350137241}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.040106535729599324, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0011179436243060426}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.20159596944281089, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0036122061036167187}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.06234392347539004, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0013928822038245677}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.01571291807548398, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006433797121782759}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07190463157595857, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002204303662857119}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.02397153359026078, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008292654015140437}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.03870233475669446, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010588745067226968}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.19464293645514244, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003469699527046509}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.06021724808437657, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013459098336822978}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.038217340553490825, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001077208240113436}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.1902899184496612, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00339164585372067}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.05928914046189273, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013399615128428988}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9686527ae2a0878c8b824007eadee083617d9561 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.11726441892112459, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.015334082664115558}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.04043384650335457, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0011680539728058876}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.20077044636212277, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037717120189089287}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.06251485774926024, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014333674940962343}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.016019666538721798, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006963432416328132}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.0727256324755699, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002280829272562891}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.024261554557884868, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008527386596796517}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.03899756530778425, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011238312786531195}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.1937695228837636, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003629063622025122}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.060317175460154866, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001386551318301422}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.03848740805974913, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011314090702100065}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.18913455692813624, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0035362816108656862}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.05935190100165203, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013786302612462235}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a61fd5dcd1c0bb79d50382df23772005032b258f --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.08326649494563637, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014667171370064546}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.13294991530121725, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0020647880464053562}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09387610190938273, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014449391906629413}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.008186684744741895, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00038085725309630776}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.014292196596118667, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007076897173253386}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.009443311010258953, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00042033592017684524}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.0745208321321628, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001231885685841352}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.12093384222921329, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018396897681209818}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0844430674301303, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001222316625035649}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.07841758933682781, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001356652627657671}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.12608243017908285, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001944196318168353}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.0886261141852728, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001340925411956822}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.3966713021013514, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.032107788039151756}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2f6d08f37f904e926a9738e3b8bfd130d69282dc --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.08502557604828932, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001409901535332923}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.13428782095164568, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0019616613248484315}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09647757726185396, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001404507794977568}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.006966193817250453, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00036778884322042845}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.011375012445045086, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000676347637821329}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.007807306910940247, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00039567014414648163}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.06912434561132932, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001024340907960514}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.11205317271963983, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0015505210767611621}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.07911103532517645, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010225471911468663}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.07928679229201335, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012924928074258797}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.1262165012516492, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0018367956599135702}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09021046038046152, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0012921693055157244}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.4340075986972079, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03441283754851664}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..7a855662900ab692c3485aa8d56a832450c2441b --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.09291365592791448, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001460793706858565}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.1488422559797642, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0020705710639246084}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.10613644765247322, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001474016161663567}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.009120310744751918, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00039466417821083983}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.015568763230043758, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007415274403876562}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.01060360006228834, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004521212932223968}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07481084575169951, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00106226725810403}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.12310873652544924, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0016574231918608155}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08624153647417561, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010877894099860167}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.0867915790156753, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013479240543794895}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.1397393265846199, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019277451004408135}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09931624380922659, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013624277073018434}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.5714321541557378, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05207006664300423}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ebda6b3177b75626d0a725ff2aa1ef4732b5fc95 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.08884062689609729, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001694039009927059}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.13695639134668142, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002297222277021766}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09749330991596093, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015999474421077564}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.010317627380981088, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004915667777414832}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.015891105747890096, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000746681384466998}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.01117408015693819, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004880353592289108}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07195531593720278, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012912975208910194}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.11407921894991604, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018782347124531222}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.07970227332043539, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012263741122186553}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.08234443508131861, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015607302462157188}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.12783583272388502, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002147908605548973}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09049100197741863, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014684127049705313}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.6781370863046939, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03589538326068359}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..bc8d1f78e303b2b565cea381ce04b6bb24e56075 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.03178506199834185, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001343108102535887}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.04796215079248023, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0018554619120391019}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.03363544745062649, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012909131114665494}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.003845960090916275, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00032358530255716713}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.006262417812252477, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0005419113367376851}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0042011360449053375, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0003274215003265904}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.026681512204475434, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011001052380045396}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.041081679608834486, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0015755732782817113}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.028264152864206495, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010441583660293569}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.02934075974862999, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012418826658471013}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.04429278174325575, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0017157301035406875}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.03091556784129967, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0011741644318024522}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.1536072771190509, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01767754971066294}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..4af2ef885a41a50896bb0f0a494b2905121c94ff --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.004982978359031628, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0005510614976068719}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.008004146641079459, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.000886422861824466}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.005186538590453919, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0005298157893824338}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0005839050846852418, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00012587591454986364}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0013034289489588643, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00039703429859922704}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0006129505049971475, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00012136340321994513}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.004267497431002324, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0004621334041194036}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.0070849664096893075, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0007922992743738505}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.004489388399312842, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0004482804717777068}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.004528845080105699, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0005065107975618541}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.007377839412377322, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0008304090072360563}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.0047159872335650254, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0004824536058253248}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.6675204478839374e-07, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 3.1908739777991254e-07}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..d2266af41abf6aef274a91dc4102abc5fd9e508e --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 0.0, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.0001111111111111111, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00011111111111110651}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 5.1282051282051286e-05, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 5.128205128205036e-05}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 7.017543859649122e-05, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 7.01754385964955e-05}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.0, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.0, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.0, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.0001111111111111111, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00011111111111110651}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 5.1282051282051286e-05, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 5.128205128205036e-05}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 7.017543859649122e-05, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 7.01754385964955e-05}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.0001111111111111111, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00011111111111110651}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 5.1282051282051286e-05, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 5.128205128205036e-05}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 7.017543859649122e-05, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 7.01754385964955e-05}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..a2052ce187891e6b2552bf6b5e06a00602d647dc --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 2.505657804731795, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08448129330294266}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.2842523517699498, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0036605089796832337}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.21222024181682922, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002811877975435317}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.22515985559604443, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00271913604007141}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.07600913474511695, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002249723324011366}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.053107638913185144, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0012617608610165124}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.05651165965956718, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012897141937349183}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2157915985634754, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0030386856144986054}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.15690291192122693, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002118913610738115}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.16712068957307108, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0020525400121234427}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.23794716782068137, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0032984804753267384}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.1745112707430124, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023783566030993763}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.1858300404558237, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002325463614772228}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..00c8d792a79e1ad77bc7322869c48571b877e4b1 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 1.4491558022298123, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11677284059045918}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.2201653187301119, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004149940282223749}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1568229775570194, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029810729084815647}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.1661375380722948, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0029609391315106206}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.06340195645403277, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002578554366386417}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.03935514148515296, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011857326939533532}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.04173045009950831, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012076024017352199}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.1713871764712513, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0034976614480037205}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.11716713627352698, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022315452547388387}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.12476790715224081, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0022207952377841602}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.1863421045122678, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0037210079404938544}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.12949037161998142, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025070704307576055}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.13765149537351365, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024972853701660013}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ee19f5d5daa46ee8dcf29aeb6aa5d86f01da58b3 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 1.0325170083670192, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09155070159493152}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.20550875384522863, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0043850189033791806}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1326798020968605, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002874666802950118}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.1441917973906388, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002913542035143854}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.06261701191581304, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002785773123682906}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.03406135061551867, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011419084537411385}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.036857023400034224, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011646994165157424}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.16608649290312155, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.003830938059547555}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.10219810669843173, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002215580186273407}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.11164498037115686, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0022462708577540924}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.17865049094772503, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004013254974233051}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.11190357275012455, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002444788908100573}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.12207928897340385, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024841685357783203}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..55dfad628b97fc34b2b90ab9b3b6477c6460cd38 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 1.134167256140814, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09717606546886882}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.22909638360887752, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004632029151207689}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1376768668142333, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002850489300883519}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.1525785492028794, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002923087860358927}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.07440792663345197, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0032019898187815683}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.0363659394734723, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011645151702284}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.0401771744546227, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012133220144334815}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.1883176484638121, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0041502761353747275}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.10660833450522245, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022044580465938987}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.11918515988014529, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0022752870246164156}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2012193449008338, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004320199440964196}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.11630225750788287, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024288025364187717}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.129729548382001, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002508780371362807}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..cb75ee8967bb8ec707d0ed554df91e2f26dcb5e1 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 1.1360525639294996, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0860200220653462}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.23672530803742303, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.004759457434672915}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1412752859993294, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002871463092526642}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.15733151178469185, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0029744474367067995}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.0807897134779241, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0034137441942331476}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.036442980899354716, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011296048388216163}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.04108775721036089, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012181280627287497}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.19449993295677506, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.004259405751648249}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.10963957627610135, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002205873184274684}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.12310488343683532, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002309424803007831}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.20742302690599165, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.004423367405689027}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.11944436234394012, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002441502584100248}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.13370390032101528, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00254631083155397}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_0.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..6fef5548c9bdbcf0caf4ebcc6e4aaff82562ffa3 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.083586610509537, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001440762933833016}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2070947923042907, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0032558812336387854}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.11685859723410175, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018776410036142014}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011753995351515174, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005738763631080483}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.030656799939853128, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015084197234712328}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.01676685311987567, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008116939067346654}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07650333684239294, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012273067013688272}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.1906362003380271, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0028813349712405664}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10717031246925551, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016214726366478817}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06525567378934424, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011000354258816365}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.16315360866507916, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026072800176013687}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09135836009153098, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014400202436564515}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.5153278620265127, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.057252202800862795}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_1.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f4a182100e81204c4cefc6f2d9e87dceb5d09925 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08126923644752823, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013883046856172049}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.20202344787769308, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003281264570631814}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.11434185937890248, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018809140186798994}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011454919678657324, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005726609882831751}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.029865905993707347, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015330029947613294}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.01631323028144093, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008130028904452307}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07392178960902045, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012059856726637796}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18437150774997185, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029021081747433526}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10408841638064358, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016358693720022312}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06395968595013059, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001077632692936246}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.16048243217413274, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026288626085393357}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09016934798656069, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001463003122509554}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.5157175151500965, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06374111661980116}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_2.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e49b50b78da7e4948ee0e833be261dcdff30f5b1 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0796828677477615, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013416709097367066}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.19965767912316895, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003229270721434028}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.11248136961296866, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018348661538257311}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.01056816485435896, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005450902704398356}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.02805644943459805, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014983600532259245}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.015174398982269022, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007849834892871412}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0726102871170291, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011735561178869995}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18252188809890302, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002873571643678364}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10258401216745326, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001610972598460782}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.061649077224170956, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0010164196126269186}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.1564969783302787, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002608152410609126}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.08732410431235793, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014130027217163148}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.46599495916506006, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.049893191051982456}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_3.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9a52a73f1fa5c9fea9428020bc7bfb98da239608 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.07901849184085133, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00158149141781139}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.1876818339260757, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0033845620592438553}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.10814433808409338, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019844919195964802}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.01086933086332831, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005902700151303653}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.026805792401099832, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014700242022978466}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.015074324129809476, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008066709490325949}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07143623535597288, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013552097346412838}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.17061868147380008, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029548218008992}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.09796289395184513, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017013034910252584}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06192685725479998, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012248997834784666}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.14788395132010507, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026431485952650934}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0847515424556008, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015104963660069898}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.5312285255112209, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.055769679108516046}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_4.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..06a394d495c95d9d8cf9ae69586b6b9d74320a2c --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.028569197114255446, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0021093926269611083}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.049986180880828145, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029038586809606247}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.03205575506161488, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001848822780502997}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.003992138336784255, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000625486280928393}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.007143856468458586, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000775527207174529}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.004447159394007513, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004831807517365398}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.02507402908479647, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0018121877476027467}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.04502723528235586, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025970015985154675}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.02857170837956627, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016148473802429979}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.02280092098484319, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017633844651269714}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.039318241724393174, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0022687079801898538}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.02532162552531629, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001460571687341891}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.2171794656385788, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0580106361575993}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_5.json b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9936d19476f9de7c10453bc23cc6d578090f36b8 --- /dev/null +++ b/1b11b51b5/evaluation/generation/agg.1b11b51b5_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0025571229514947014, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007281415062359935}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0021566578919806367, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0005911769409561811}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0022764186005474617, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0006298696283878379}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.00014412997903563943, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 8.348859628883554e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.00011864996969267079, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 6.932933515769192e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0001281030666243269, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 7.404161859751549e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0018849622542887335, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.000531970687877808}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.001615365856940405, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0004314744133837763}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0016885822290001135, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0004582016072242166}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.001995793577880732, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0005540133520309812}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0017756009214037463, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.000488425634026763}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0018318538559767147, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005037674635893385}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.2989205700146707e-37, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.6500888245792048e-32}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..93f8b9cc88fdbca2668125953d62b4ae6912efc0 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:199b843e1d5bca3ad87599f7c2bac05a547268bba6613ac8585e8bf9f3480697 +size 4142451 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4f29eb40a58c91aa7704eaff3136bba3644317c7 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab1c64cbe4fdcaced0ffb3fa4352c39cd0a6d1e05faf8039e3abc22ca806a58 +size 5237784 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2ce953ad2586b0506c671ce367fb2621f28c4fb9 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d042353d3c946859792c4cc3a85b40a5bcddb9c17df580106056d735530d9fba +size 6120920 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b4710eb063201312c6d4152b801f3659e7bf82bd --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:592bba107623c7aa25cf4e873da20a19addf7a51302ec1ec8c045a7212e440d2 +size 7024897 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..88e12896cc5cecd1d5df08f3320209ca7816f93a --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d88fa9da595c86a17f99595b91d138d0599b0f77728b618261796b4ae5b06240 +size 7909953 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..30f710ce8248070c9714e85046d146e3a8834b2f --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83afe593fff2febb4a53d46c4dd4a6d8e23cd797f276ea09a1711e964815dfd6 +size 8791715 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ec1ade21804602655e953d5537bd6b311a1e4c28 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:defc0c8170fb4b50fe692ede2285195d39480dff0adf04ae8eea5121d5813355 +size 7642798 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8dbda3e85639f30ae2274c8f31402ade408301e2 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:962306b5d258b8fe49b79021b43c38fd366008098bf274891735a59944f4bc98 +size 13329154 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8eae88873d7f22b33303ff48d132d19816567d01 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5163617abcccf738a831dac64fd5d458d85d993090adc3c174b7276736ae0801 +size 18927352 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e269b3e6b51dc003428a278af6aa9b6e46e624fa --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:560252f3f81b2f99fb4ab35f3fce6f6bbf5d1120aad993cfa70904c89e306e32 +size 24339563 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2c03190107dcac8ec33d7c7bee17ad62a5444370 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3ad9a66a3a05d9728da8b5976ea86dd42604f5d1b18935ea1b1af50b9089902 +size 29473275 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d02440c0c1d9c439c5ec8f5e877adde4cc0c1caf --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee85847a4e8aa0a60f4086fa9c4236e1ed155abd805229eaa3beccc5adb856d +size 34799581 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..67d84082d328f43b3621fc4e00d823c6b59d0893 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c76f9001e673521838061f4f3e8621acf2673f250836863d0069288091e73d4c +size 3619879 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3a8bc49bf433e2a0cc523c5be8568abd228dde39 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16eaf876638dda122fd8e227b0a18e7ba8bacbce316e131975f3a7f794253369 +size 4949726 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3129a5daf7c07289575c920e03fc790b3bc64686 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba321e519518ea6d2bee58d41f4e25c88b77f34eea465a5d27cccbd3a755f782 +size 5966712 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..60046629b6f42d601a3e88b231063f33d0187c3a --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e9e267da6bca04744c1b8f452e3476745bae8718723a4b6fe0fd7eb547c4752 +size 7019053 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3900819b8300cc014f90048dea40c8418bcd185f --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6401eb9b71f2f643ad1d6d1437953a7406cd01b460a4462a38ef7610dd5bfc9d +size 8098691 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4d8419f130c16f42631d93194ff3040037403f32 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363515084d8a39d93b6d4f10e9d5eaea1a3122ad1e9c2a39ad53f7b2aa3de157 +size 9189226 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_0.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..feb620638420ad052b8a209716b677c407244316 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9a6fb3e978af84c6cc3233a253ae48b36618f5c792d8d38726c9f9612256d7a +size 2826646 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_1.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fe1b3e3247ab90ec8541ba96a5b421c867bbb139 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:214fe22efae3c62e968ab97b581d747e80ee2d84893d1f85a9e907309380125b +size 5095719 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_2.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8e86b90f37b7cf0055448e7a2246f37c6a310974 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e4193c877eb3f262007d266407d0a4df2316e073755c1ec85ea1cc3ec9aa985 +size 7369776 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_3.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7cf1d90be340dbfdcdf2dbf4ec134842f36c45ac --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c054f0879214cfcd5d8813780529076a2ed620104292ad9c271dc334dcb6d415 +size 9639352 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_4.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bce5d30ad473529de6b2bc06cf7d031952f05650 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ffe8f80add0344fe6daabe8f48ec441884733e891ce2dce1799703e2a655d38 +size 11671150 diff --git a/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_5.jsonl b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..222fbdb573e81a67d64d19622cabd12cca2d9271 --- /dev/null +++ b/1b11b51b5/evaluation/generation/examples.1b11b51b5_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d728e27c379c152fa1f968a89fa6ec2358660f42fd5d6690c871ac2dd1d633e +size 13897539 diff --git a/1b11b51b5/evaluation/generation/merged.csv b/1b11b51b5/evaluation/generation/merged.csv new file mode 100644 index 0000000000000000000000000000000000000000..41dcb312fb8879e6caf14310e7b4d9c5c4497db2 --- /dev/null +++ b/1b11b51b5/evaluation/generation/merged.csv @@ -0,0 +1,53 @@ +dataset,fewshots,prompt,metric,value +e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.0 +e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.0 +e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.05651165965956718 +e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.05651165965956718 +e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.04173045009950831 +e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.04173045009950831 +e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.036857023400034224 +e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.036857023400034224 +e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.0401771744546227 +e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.0401771744546227 +e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.04108775721036089 +e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.04108775721036089 +e2e_nlg_cleaned,5,average,multiple,0.036060677470682215 +gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.01676685311987567 +gem_xsum,0,median,rouge2_fmeasure,0.01676685311987567 +gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.01631323028144093 +gem_xsum,1,median,rouge2_fmeasure,0.01631323028144093 +gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.015174398982269022 +gem_xsum,2,median,rouge2_fmeasure,0.015174398982269022 +gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.015074324129809476 +gem_xsum,3,median,rouge2_fmeasure,0.015074324129809476 +gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.004447159394007513 +gem_xsum,4,median,rouge2_fmeasure,0.004447159394007513 +gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.0001281030666243269 +gem_xsum,5,median,rouge2_fmeasure,0.0001281030666243269 +gem_xsum,5,average,multiple,0.01131734482900449 +web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.03650350909082802 +web_nlg_en,0,median,rouge2_fmeasure,0.03650350909082802 +web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.025933814076285186 +web_nlg_en,1,median,rouge2_fmeasure,0.025933814076285186 +web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.024990803032608512 +web_nlg_en,2,median,rouge2_fmeasure,0.024990803032608512 +web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.0250870194649846 +web_nlg_en,3,median,rouge2_fmeasure,0.0250870194649846 +web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.02397153359026078 +web_nlg_en,4,median,rouge2_fmeasure,0.02397153359026078 +web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.024261554557884868 +web_nlg_en,5,median,rouge2_fmeasure,0.024261554557884868 +web_nlg_en,5,average,multiple,0.026791372302141996 +wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.009443311010258953 +wiki_lingua_en,0,median,rouge2_fmeasure,0.009443311010258953 +wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.007807306910940247 +wiki_lingua_en,1,median,rouge2_fmeasure,0.007807306910940247 +wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.01060360006228834 +wiki_lingua_en,2,median,rouge2_fmeasure,0.01060360006228834 +wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.01117408015693819 +wiki_lingua_en,3,median,rouge2_fmeasure,0.01117408015693819 +wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.0042011360449053375 +wiki_lingua_en,4,median,rouge2_fmeasure,0.0042011360449053375 +wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.0006129505049971475 +wiki_lingua_en,5,median,rouge2_fmeasure,0.0006129505049971475 +wiki_lingua_en,5,average,multiple,0.007307064115054702 diff --git a/1b11b51b5/evaluation/generation/merged.json b/1b11b51b5/evaluation/generation/merged.json new file mode 100644 index 0000000000000000000000000000000000000000..e41166df4421b32b7172ec2dfa2d82c43f929125 --- /dev/null +++ b/1b11b51b5/evaluation/generation/merged.json @@ -0,0 +1 @@ +{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.22551871811397353, "bleu_stderr": 0.02136035868888881, "rouge1_fmeasure": 0.08179555310979232, "rouge1_fmeasure_stderr": 0.0015855832653346308, "rouge1_precision": 0.05258955980824922, "rouge1_precision_stderr": 0.001111043010483631, "rouge1_recall": 0.24440401749751364, "rouge1_recall_stderr": 0.004069008614986987, "rouge2_fmeasure": 0.03650350909082802, "rouge2_fmeasure_stderr": 0.0010163354215107154, "rouge2_precision": 0.023193335272543247, "rouge2_precision_stderr": 0.0006829041826176187, "rouge2_recall": 0.11105971904247877, "rouge2_recall_stderr": 0.0028341010971909687, "rougeL_fmeasure": 0.0797122178196054, "rougeL_fmeasure_stderr": 0.0015418999962278706, "rougeL_precision": 0.051210428458407814, "rougeL_precision_stderr": 0.0010759000779014146, "rougeL_recall": 0.23873249693202636, "rougeL_recall_stderr": 0.004010114948451509, "rougeLsum_fmeasure": 0.07835236496419286, "rougeLsum_fmeasure_stderr": 0.0015149830080257896, "rougeLsum_precision": 0.050443394209883134, "rougeLsum_precision_stderr": 0.0010665847609739637, "rougeLsum_recall": 0.23354882611756766, "rougeLsum_recall_stderr": 0.0038288448419567423}}, "1": {"PALM_prompt": {"bleu": 0.10303247472010832, "bleu_stderr": 0.010542121107444794, "rouge1_fmeasure": 0.06582659671546148, "rouge1_fmeasure_stderr": 0.0014010035101277287, "rouge1_precision": 0.041949850500467546, "rouge1_precision_stderr": 0.001009545354318454, "rouge1_recall": 0.21524248765770787, "rouge1_recall_stderr": 0.003637995924121415, "rouge2_fmeasure": 0.025933814076285186, "rouge2_fmeasure_stderr": 0.000854986393522763, "rouge2_precision": 0.016602402097909204, "rouge2_precision_stderr": 0.0005784273046205441, "rouge2_recall": 0.0786634484231439, "rouge2_recall_stderr": 0.002376415534094953, "rougeL_fmeasure": 0.0644436752805007, "rougeL_fmeasure_stderr": 0.001375438208183654, "rougeL_precision": 0.041053525063470375, "rougeL_precision_stderr": 0.0009905003168813127, "rougeL_recall": 0.21058233480126393, "rougeL_recall_stderr": 0.0035581210952122444, "rougeLsum_fmeasure": 0.06327032653279559, "rougeLsum_fmeasure_stderr": 0.0013696218498744507, "rougeLsum_precision": 0.040395747717907556, "rougeLsum_precision_stderr": 0.0009891799118591429, "rougeLsum_recall": 0.20549607203927084, "rougeLsum_recall_stderr": 0.0034995349438813944}}, "2": {"PALM_prompt": {"bleu": 0.10044377394916508, "bleu_stderr": 0.011004903600055578, "rouge1_fmeasure": 0.06380975510656396, "rouge1_fmeasure_stderr": 0.0013921200872663029, "rouge1_precision": 0.04071859925486269, "rouge1_precision_stderr": 0.000989754132711725, "rouge1_recall": 0.2071041537148272, "rouge1_recall_stderr": 0.0036456928364795613, "rouge2_fmeasure": 0.024990803032608512, "rouge2_fmeasure_stderr": 0.0008536543867796345, "rouge2_precision": 0.016062068279265852, "rouge2_precision_stderr": 0.0005807608561522497, "rouge2_recall": 0.07481502747530522, "rouge2_recall_stderr": 0.002374523180072443, "rougeL_fmeasure": 0.06228298370655369, "rougeL_fmeasure_stderr": 0.0013594322207689349, "rougeL_precision": 0.03972537172680168, "rougeL_precision_stderr": 0.0009640633401932609, "rougeL_recall": 0.20198143435079377, "rougeL_recall_stderr": 0.0035579835868813966, "rougeLsum_fmeasure": 0.06120828937572442, "rougeLsum_fmeasure_stderr": 0.0013588698800721009, "rougeLsum_precision": 0.039139482148406846, "rougeLsum_precision_stderr": 0.0009678826987632891, "rougeLsum_recall": 0.19661295375267737, "rougeLsum_recall_stderr": 0.0034609252034809553}}, "3": {"PALM_prompt": {"bleu": 0.11695012147806128, "bleu_stderr": 0.009564035174544912, "rouge1_fmeasure": 0.06398897669967853, "rouge1_fmeasure_stderr": 0.0014297013856788472, "rouge1_precision": 0.040613137406636475, "rouge1_precision_stderr": 0.000993472498566677, "rouge1_recall": 0.20769491305475654, "rouge1_recall_stderr": 0.003694525077541037, "rouge2_fmeasure": 0.0250870194649846, "rouge2_fmeasure_stderr": 0.0008636090708303351, "rouge2_precision": 0.016119706774984935, "rouge2_precision_stderr": 0.0005886522490528559, "rouge2_recall": 0.07411626939385718, "rouge2_recall_stderr": 0.0022817865110913976, "rougeL_fmeasure": 0.06200730116971786, "rougeL_fmeasure_stderr": 0.0013847498199754179, "rougeL_precision": 0.03934321475895627, "rougeL_precision_stderr": 0.0009617791254833742, "rougeL_recall": 0.2011219485070139, "rougeL_recall_stderr": 0.0035514042056288375, "rougeLsum_fmeasure": 0.060903726984725515, "rougeLsum_fmeasure_stderr": 0.001384286630928201, "rougeLsum_precision": 0.03873209326674636, "rougeLsum_precision_stderr": 0.0009654074102020647, "rougeLsum_recall": 0.196124698805037, "rougeLsum_recall_stderr": 0.0034935507768228203}}, "4": {"PALM_prompt": {"bleu": 0.09979909326915241, "bleu_stderr": 0.015027652350137241, "rouge1_fmeasure": 0.06234392347539004, "rouge1_fmeasure_stderr": 0.0013928822038245677, "rouge1_precision": 0.040106535729599324, "rouge1_precision_stderr": 0.0011179436243060426, "rouge1_recall": 0.20159596944281089, "rouge1_recall_stderr": 0.0036122061036167187, "rouge2_fmeasure": 0.02397153359026078, "rouge2_fmeasure_stderr": 0.0008292654015140437, "rouge2_precision": 0.01571291807548398, "rouge2_precision_stderr": 0.0006433797121782759, "rouge2_recall": 0.07190463157595857, "rouge2_recall_stderr": 0.002204303662857119, "rougeL_fmeasure": 0.06021724808437657, "rougeL_fmeasure_stderr": 0.0013459098336822978, "rougeL_precision": 0.03870233475669446, "rougeL_precision_stderr": 0.0010588745067226968, "rougeL_recall": 0.19464293645514244, "rougeL_recall_stderr": 0.003469699527046509, "rougeLsum_fmeasure": 0.05928914046189273, "rougeLsum_fmeasure_stderr": 0.0013399615128428988, "rougeLsum_precision": 0.038217340553490825, "rougeLsum_precision_stderr": 0.001077208240113436, "rougeLsum_recall": 0.1902899184496612, "rougeLsum_recall_stderr": 0.00339164585372067}}, "5": {"PALM_prompt": {"bleu": 0.11726441892112459, "bleu_stderr": 0.015334082664115558, "rouge1_fmeasure": 0.06251485774926024, "rouge1_fmeasure_stderr": 0.0014333674940962343, "rouge1_precision": 0.04043384650335457, "rouge1_precision_stderr": 0.0011680539728058876, "rouge1_recall": 0.20077044636212277, "rouge1_recall_stderr": 0.0037717120189089287, "rouge2_fmeasure": 0.024261554557884868, "rouge2_fmeasure_stderr": 0.0008527386596796517, "rouge2_precision": 0.016019666538721798, "rouge2_precision_stderr": 0.0006963432416328132, "rouge2_recall": 0.0727256324755699, "rouge2_recall_stderr": 0.002280829272562891, "rougeL_fmeasure": 0.060317175460154866, "rougeL_fmeasure_stderr": 0.001386551318301422, "rougeL_precision": 0.03899756530778425, "rougeL_precision_stderr": 0.0011238312786531195, "rougeL_recall": 0.1937695228837636, "rougeL_recall_stderr": 0.003629063622025122, "rougeLsum_fmeasure": 0.05935190100165203, "rougeLsum_fmeasure_stderr": 0.0013786302612462235, "rougeLsum_precision": 0.03848740805974913, "rougeLsum_precision_stderr": 0.0011314090702100065, "rougeLsum_recall": 0.18913455692813624, "rougeLsum_recall_stderr": 0.0035362816108656862}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 0.3966713021013514, "bleu_stderr": 0.032107788039151756, "rouge1_fmeasure": 0.09387610190938273, "rouge1_fmeasure_stderr": 0.0014449391906629413, "rouge1_precision": 0.08326649494563637, "rouge1_precision_stderr": 0.0014667171370064546, "rouge1_recall": 0.13294991530121725, "rouge1_recall_stderr": 0.0020647880464053562, "rouge2_fmeasure": 0.009443311010258953, "rouge2_fmeasure_stderr": 0.00042033592017684524, "rouge2_precision": 0.008186684744741895, "rouge2_precision_stderr": 0.00038085725309630776, "rouge2_recall": 0.014292196596118667, "rouge2_recall_stderr": 0.0007076897173253386, "rougeL_fmeasure": 0.0844430674301303, "rougeL_fmeasure_stderr": 0.001222316625035649, "rougeL_precision": 0.0745208321321628, "rougeL_precision_stderr": 0.001231885685841352, "rougeL_recall": 0.12093384222921329, "rougeL_recall_stderr": 0.0018396897681209818, "rougeLsum_fmeasure": 0.0886261141852728, "rougeLsum_fmeasure_stderr": 0.001340925411956822, "rougeLsum_precision": 0.07841758933682781, "rougeLsum_precision_stderr": 0.001356652627657671, "rougeLsum_recall": 0.12608243017908285, "rougeLsum_recall_stderr": 0.001944196318168353}}, "1": {"tldr_en": {"bleu": 0.4340075986972079, "bleu_stderr": 0.03441283754851664, "rouge1_fmeasure": 0.09647757726185396, "rouge1_fmeasure_stderr": 0.001404507794977568, "rouge1_precision": 0.08502557604828932, "rouge1_precision_stderr": 0.001409901535332923, "rouge1_recall": 0.13428782095164568, "rouge1_recall_stderr": 0.0019616613248484315, "rouge2_fmeasure": 0.007807306910940247, "rouge2_fmeasure_stderr": 0.00039567014414648163, "rouge2_precision": 0.006966193817250453, "rouge2_precision_stderr": 0.00036778884322042845, "rouge2_recall": 0.011375012445045086, "rouge2_recall_stderr": 0.000676347637821329, "rougeL_fmeasure": 0.07911103532517645, "rougeL_fmeasure_stderr": 0.0010225471911468663, "rougeL_precision": 0.06912434561132932, "rougeL_precision_stderr": 0.001024340907960514, "rougeL_recall": 0.11205317271963983, "rougeL_recall_stderr": 0.0015505210767611621, "rougeLsum_fmeasure": 0.09021046038046152, "rougeLsum_fmeasure_stderr": 0.0012921693055157244, "rougeLsum_precision": 0.07928679229201335, "rougeLsum_precision_stderr": 0.0012924928074258797, "rougeLsum_recall": 0.1262165012516492, "rougeLsum_recall_stderr": 0.0018367956599135702}}, "2": {"tldr_en": {"bleu": 0.5714321541557378, "bleu_stderr": 0.05207006664300423, "rouge1_fmeasure": 0.10613644765247322, "rouge1_fmeasure_stderr": 0.001474016161663567, "rouge1_precision": 0.09291365592791448, "rouge1_precision_stderr": 0.001460793706858565, "rouge1_recall": 0.1488422559797642, "rouge1_recall_stderr": 0.0020705710639246084, "rouge2_fmeasure": 0.01060360006228834, "rouge2_fmeasure_stderr": 0.0004521212932223968, "rouge2_precision": 0.009120310744751918, "rouge2_precision_stderr": 0.00039466417821083983, "rouge2_recall": 0.015568763230043758, "rouge2_recall_stderr": 0.0007415274403876562, "rougeL_fmeasure": 0.08624153647417561, "rougeL_fmeasure_stderr": 0.0010877894099860167, "rougeL_precision": 0.07481084575169951, "rougeL_precision_stderr": 0.00106226725810403, "rougeL_recall": 0.12310873652544924, "rougeL_recall_stderr": 0.0016574231918608155, "rougeLsum_fmeasure": 0.09931624380922659, "rougeLsum_fmeasure_stderr": 0.0013624277073018434, "rougeLsum_precision": 0.0867915790156753, "rougeLsum_precision_stderr": 0.0013479240543794895, "rougeLsum_recall": 0.1397393265846199, "rougeLsum_recall_stderr": 0.0019277451004408135}}, "3": {"tldr_en": {"bleu": 0.6781370863046939, "bleu_stderr": 0.03589538326068359, "rouge1_fmeasure": 0.09749330991596093, "rouge1_fmeasure_stderr": 0.0015999474421077564, "rouge1_precision": 0.08884062689609729, "rouge1_precision_stderr": 0.001694039009927059, "rouge1_recall": 0.13695639134668142, "rouge1_recall_stderr": 0.002297222277021766, "rouge2_fmeasure": 0.01117408015693819, "rouge2_fmeasure_stderr": 0.0004880353592289108, "rouge2_precision": 0.010317627380981088, "rouge2_precision_stderr": 0.0004915667777414832, "rouge2_recall": 0.015891105747890096, "rouge2_recall_stderr": 0.000746681384466998, "rougeL_fmeasure": 0.07970227332043539, "rougeL_fmeasure_stderr": 0.0012263741122186553, "rougeL_precision": 0.07195531593720278, "rougeL_precision_stderr": 0.0012912975208910194, "rougeL_recall": 0.11407921894991604, "rougeL_recall_stderr": 0.0018782347124531222, "rougeLsum_fmeasure": 0.09049100197741863, "rougeLsum_fmeasure_stderr": 0.0014684127049705313, "rougeLsum_precision": 0.08234443508131861, "rougeLsum_precision_stderr": 0.0015607302462157188, "rougeLsum_recall": 0.12783583272388502, "rougeLsum_recall_stderr": 0.002147908605548973}}, "4": {"tldr_en": {"bleu": 0.1536072771190509, "bleu_stderr": 0.01767754971066294, "rouge1_fmeasure": 0.03363544745062649, "rouge1_fmeasure_stderr": 0.0012909131114665494, "rouge1_precision": 0.03178506199834185, "rouge1_precision_stderr": 0.001343108102535887, "rouge1_recall": 0.04796215079248023, "rouge1_recall_stderr": 0.0018554619120391019, "rouge2_fmeasure": 0.0042011360449053375, "rouge2_fmeasure_stderr": 0.0003274215003265904, "rouge2_precision": 0.003845960090916275, "rouge2_precision_stderr": 0.00032358530255716713, "rouge2_recall": 0.006262417812252477, "rouge2_recall_stderr": 0.0005419113367376851, "rougeL_fmeasure": 0.028264152864206495, "rougeL_fmeasure_stderr": 0.0010441583660293569, "rougeL_precision": 0.026681512204475434, "rougeL_precision_stderr": 0.0011001052380045396, "rougeL_recall": 0.041081679608834486, "rougeL_recall_stderr": 0.0015755732782817113, "rougeLsum_fmeasure": 0.03091556784129967, "rougeLsum_fmeasure_stderr": 0.0011741644318024522, "rougeLsum_precision": 0.02934075974862999, "rougeLsum_precision_stderr": 0.0012418826658471013, "rougeLsum_recall": 0.04429278174325575, "rougeLsum_recall_stderr": 0.0017157301035406875}}, "5": {"tldr_en": {"bleu": 1.6675204478839374e-07, "bleu_stderr": 3.1908739777991254e-07, "rouge1_fmeasure": 0.005186538590453919, "rouge1_fmeasure_stderr": 0.0005298157893824338, "rouge1_precision": 0.004982978359031628, "rouge1_precision_stderr": 0.0005510614976068719, "rouge1_recall": 0.008004146641079459, "rouge1_recall_stderr": 0.000886422861824466, "rouge2_fmeasure": 0.0006129505049971475, "rouge2_fmeasure_stderr": 0.00012136340321994513, "rouge2_precision": 0.0005839050846852418, "rouge2_precision_stderr": 0.00012587591454986364, "rouge2_recall": 0.0013034289489588643, "rouge2_recall_stderr": 0.00039703429859922704, "rougeL_fmeasure": 0.004489388399312842, "rougeL_fmeasure_stderr": 0.0004482804717777068, "rougeL_precision": 0.004267497431002324, "rougeL_precision_stderr": 0.0004621334041194036, "rougeL_recall": 0.0070849664096893075, "rougeL_recall_stderr": 0.0007922992743738505, "rougeLsum_fmeasure": 0.0047159872335650254, "rougeLsum_fmeasure_stderr": 0.0004824536058253248, "rougeLsum_precision": 0.004528845080105699, "rougeLsum_precision_stderr": 0.0005065107975618541, "rougeLsum_recall": 0.007377839412377322, "rougeLsum_recall_stderr": 0.0008304090072360563}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 0.0, "bleu_stderr": 0.0, "rouge1_fmeasure": 7.017543859649122e-05, "rouge1_fmeasure_stderr": 7.01754385964955e-05, "rouge1_precision": 0.0001111111111111111, "rouge1_precision_stderr": 0.00011111111111110651, "rouge1_recall": 5.1282051282051286e-05, "rouge1_recall_stderr": 5.128205128205036e-05, "rouge2_fmeasure": 0.0, "rouge2_fmeasure_stderr": 0.0, "rouge2_precision": 0.0, "rouge2_precision_stderr": 0.0, "rouge2_recall": 0.0, "rouge2_recall_stderr": 0.0, "rougeL_fmeasure": 7.017543859649122e-05, "rougeL_fmeasure_stderr": 7.01754385964955e-05, "rougeL_precision": 0.0001111111111111111, "rougeL_precision_stderr": 0.00011111111111110651, "rougeL_recall": 5.1282051282051286e-05, "rougeL_recall_stderr": 5.128205128205036e-05, "rougeLsum_fmeasure": 7.017543859649122e-05, "rougeLsum_fmeasure_stderr": 7.01754385964955e-05, "rougeLsum_precision": 0.0001111111111111111, "rougeLsum_precision_stderr": 0.00011111111111110651, "rougeLsum_recall": 5.1282051282051286e-05, "rougeLsum_recall_stderr": 5.128205128205036e-05}}, "1": {"generate_text_restaurant": {"bleu": 2.505657804731795, "bleu_stderr": 0.08448129330294266, "rouge1_fmeasure": 0.22515985559604443, "rouge1_fmeasure_stderr": 0.00271913604007141, "rouge1_precision": 0.2842523517699498, "rouge1_precision_stderr": 0.0036605089796832337, "rouge1_recall": 0.21222024181682922, "rouge1_recall_stderr": 0.002811877975435317, "rouge2_fmeasure": 0.05651165965956718, "rouge2_fmeasure_stderr": 0.0012897141937349183, "rouge2_precision": 0.07600913474511695, "rouge2_precision_stderr": 0.002249723324011366, "rouge2_recall": 0.053107638913185144, "rouge2_recall_stderr": 0.0012617608610165124, "rougeL_fmeasure": 0.16712068957307108, "rougeL_fmeasure_stderr": 0.0020525400121234427, "rougeL_precision": 0.2157915985634754, "rougeL_precision_stderr": 0.0030386856144986054, "rougeL_recall": 0.15690291192122693, "rougeL_recall_stderr": 0.002118913610738115, "rougeLsum_fmeasure": 0.1858300404558237, "rougeLsum_fmeasure_stderr": 0.002325463614772228, "rougeLsum_precision": 0.23794716782068137, "rougeLsum_precision_stderr": 0.0032984804753267384, "rougeLsum_recall": 0.1745112707430124, "rougeLsum_recall_stderr": 0.0023783566030993763}}, "2": {"generate_text_restaurant": {"bleu": 1.4491558022298123, "bleu_stderr": 0.11677284059045918, "rouge1_fmeasure": 0.1661375380722948, "rouge1_fmeasure_stderr": 0.0029609391315106206, "rouge1_precision": 0.2201653187301119, "rouge1_precision_stderr": 0.004149940282223749, "rouge1_recall": 0.1568229775570194, "rouge1_recall_stderr": 0.0029810729084815647, "rouge2_fmeasure": 0.04173045009950831, "rouge2_fmeasure_stderr": 0.0012076024017352199, "rouge2_precision": 0.06340195645403277, "rouge2_precision_stderr": 0.002578554366386417, "rouge2_recall": 0.03935514148515296, "rouge2_recall_stderr": 0.0011857326939533532, "rougeL_fmeasure": 0.12476790715224081, "rougeL_fmeasure_stderr": 0.0022207952377841602, "rougeL_precision": 0.1713871764712513, "rougeL_precision_stderr": 0.0034976614480037205, "rougeL_recall": 0.11716713627352698, "rougeL_recall_stderr": 0.0022315452547388387, "rougeLsum_fmeasure": 0.13765149537351365, "rougeLsum_fmeasure_stderr": 0.0024972853701660013, "rougeLsum_precision": 0.1863421045122678, "rougeLsum_precision_stderr": 0.0037210079404938544, "rougeLsum_recall": 0.12949037161998142, "rougeLsum_recall_stderr": 0.0025070704307576055}}, "3": {"generate_text_restaurant": {"bleu": 1.0325170083670192, "bleu_stderr": 0.09155070159493152, "rouge1_fmeasure": 0.1441917973906388, "rouge1_fmeasure_stderr": 0.002913542035143854, "rouge1_precision": 0.20550875384522863, "rouge1_precision_stderr": 0.0043850189033791806, "rouge1_recall": 0.1326798020968605, "rouge1_recall_stderr": 0.002874666802950118, "rouge2_fmeasure": 0.036857023400034224, "rouge2_fmeasure_stderr": 0.0011646994165157424, "rouge2_precision": 0.06261701191581304, "rouge2_precision_stderr": 0.002785773123682906, "rouge2_recall": 0.03406135061551867, "rouge2_recall_stderr": 0.0011419084537411385, "rougeL_fmeasure": 0.11164498037115686, "rougeL_fmeasure_stderr": 0.0022462708577540924, "rougeL_precision": 0.16608649290312155, "rougeL_precision_stderr": 0.003830938059547555, "rougeL_recall": 0.10219810669843173, "rougeL_recall_stderr": 0.002215580186273407, "rougeLsum_fmeasure": 0.12207928897340385, "rougeLsum_fmeasure_stderr": 0.0024841685357783203, "rougeLsum_precision": 0.17865049094772503, "rougeLsum_precision_stderr": 0.004013254974233051, "rougeLsum_recall": 0.11190357275012455, "rougeLsum_recall_stderr": 0.002444788908100573}}, "4": {"generate_text_restaurant": {"bleu": 1.134167256140814, "bleu_stderr": 0.09717606546886882, "rouge1_fmeasure": 0.1525785492028794, "rouge1_fmeasure_stderr": 0.002923087860358927, "rouge1_precision": 0.22909638360887752, "rouge1_precision_stderr": 0.004632029151207689, "rouge1_recall": 0.1376768668142333, "rouge1_recall_stderr": 0.002850489300883519, "rouge2_fmeasure": 0.0401771744546227, "rouge2_fmeasure_stderr": 0.0012133220144334815, "rouge2_precision": 0.07440792663345197, "rouge2_precision_stderr": 0.0032019898187815683, "rouge2_recall": 0.0363659394734723, "rouge2_recall_stderr": 0.0011645151702284, "rougeL_fmeasure": 0.11918515988014529, "rougeL_fmeasure_stderr": 0.0022752870246164156, "rougeL_precision": 0.1883176484638121, "rougeL_precision_stderr": 0.0041502761353747275, "rougeL_recall": 0.10660833450522245, "rougeL_recall_stderr": 0.0022044580465938987, "rougeLsum_fmeasure": 0.129729548382001, "rougeLsum_fmeasure_stderr": 0.002508780371362807, "rougeLsum_precision": 0.2012193449008338, "rougeLsum_precision_stderr": 0.004320199440964196, "rougeLsum_recall": 0.11630225750788287, "rougeLsum_recall_stderr": 0.0024288025364187717}}, "5": {"generate_text_restaurant": {"bleu": 1.1360525639294996, "bleu_stderr": 0.0860200220653462, "rouge1_fmeasure": 0.15733151178469185, "rouge1_fmeasure_stderr": 0.0029744474367067995, "rouge1_precision": 0.23672530803742303, "rouge1_precision_stderr": 0.004759457434672915, "rouge1_recall": 0.1412752859993294, "rouge1_recall_stderr": 0.002871463092526642, "rouge2_fmeasure": 0.04108775721036089, "rouge2_fmeasure_stderr": 0.0012181280627287497, "rouge2_precision": 0.0807897134779241, "rouge2_precision_stderr": 0.0034137441942331476, "rouge2_recall": 0.036442980899354716, "rouge2_recall_stderr": 0.0011296048388216163, "rougeL_fmeasure": 0.12310488343683532, "rougeL_fmeasure_stderr": 0.002309424803007831, "rougeL_precision": 0.19449993295677506, "rougeL_precision_stderr": 0.004259405751648249, "rougeL_recall": 0.10963957627610135, "rougeL_recall_stderr": 0.002205873184274684, "rougeLsum_fmeasure": 0.13370390032101528, "rougeLsum_fmeasure_stderr": 0.00254631083155397, "rougeLsum_precision": 0.20742302690599165, "rougeLsum_precision_stderr": 0.004423367405689027, "rougeLsum_recall": 0.11944436234394012, "rougeLsum_recall_stderr": 0.002441502584100248}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 0.5153278620265127, "bleu_stderr": 0.057252202800862795, "rouge1_fmeasure": 0.11685859723410175, "rouge1_fmeasure_stderr": 0.0018776410036142014, "rouge1_precision": 0.083586610509537, "rouge1_precision_stderr": 0.001440762933833016, "rouge1_recall": 0.2070947923042907, "rouge1_recall_stderr": 0.0032558812336387854, "rouge2_fmeasure": 0.01676685311987567, "rouge2_fmeasure_stderr": 0.0008116939067346654, "rouge2_precision": 0.011753995351515174, "rouge2_precision_stderr": 0.0005738763631080483, "rouge2_recall": 0.030656799939853128, "rouge2_recall_stderr": 0.0015084197234712328, "rougeL_fmeasure": 0.10717031246925551, "rougeL_fmeasure_stderr": 0.0016214726366478817, "rougeL_precision": 0.07650333684239294, "rougeL_precision_stderr": 0.0012273067013688272, "rougeL_recall": 0.1906362003380271, "rougeL_recall_stderr": 0.0028813349712405664, "rougeLsum_fmeasure": 0.09135836009153098, "rougeLsum_fmeasure_stderr": 0.0014400202436564515, "rougeLsum_precision": 0.06525567378934424, "rougeLsum_precision_stderr": 0.0011000354258816365, "rougeLsum_recall": 0.16315360866507916, "rougeLsum_recall_stderr": 0.0026072800176013687}}, "1": {"article_DOC_summary": {"bleu": 0.5157175151500965, "bleu_stderr": 0.06374111661980116, "rouge1_fmeasure": 0.11434185937890248, "rouge1_fmeasure_stderr": 0.0018809140186798994, "rouge1_precision": 0.08126923644752823, "rouge1_precision_stderr": 0.0013883046856172049, "rouge1_recall": 0.20202344787769308, "rouge1_recall_stderr": 0.003281264570631814, "rouge2_fmeasure": 0.01631323028144093, "rouge2_fmeasure_stderr": 0.0008130028904452307, "rouge2_precision": 0.011454919678657324, "rouge2_precision_stderr": 0.0005726609882831751, "rouge2_recall": 0.029865905993707347, "rouge2_recall_stderr": 0.0015330029947613294, "rougeL_fmeasure": 0.10408841638064358, "rougeL_fmeasure_stderr": 0.0016358693720022312, "rougeL_precision": 0.07392178960902045, "rougeL_precision_stderr": 0.0012059856726637796, "rougeL_recall": 0.18437150774997185, "rougeL_recall_stderr": 0.0029021081747433526, "rougeLsum_fmeasure": 0.09016934798656069, "rougeLsum_fmeasure_stderr": 0.001463003122509554, "rougeLsum_precision": 0.06395968595013059, "rougeLsum_precision_stderr": 0.001077632692936246, "rougeLsum_recall": 0.16048243217413274, "rougeLsum_recall_stderr": 0.0026288626085393357}}, "2": {"article_DOC_summary": {"bleu": 0.46599495916506006, "bleu_stderr": 0.049893191051982456, "rouge1_fmeasure": 0.11248136961296866, "rouge1_fmeasure_stderr": 0.0018348661538257311, "rouge1_precision": 0.0796828677477615, "rouge1_precision_stderr": 0.0013416709097367066, "rouge1_recall": 0.19965767912316895, "rouge1_recall_stderr": 0.003229270721434028, "rouge2_fmeasure": 0.015174398982269022, "rouge2_fmeasure_stderr": 0.0007849834892871412, "rouge2_precision": 0.01056816485435896, "rouge2_precision_stderr": 0.0005450902704398356, "rouge2_recall": 0.02805644943459805, "rouge2_recall_stderr": 0.0014983600532259245, "rougeL_fmeasure": 0.10258401216745326, "rougeL_fmeasure_stderr": 0.001610972598460782, "rougeL_precision": 0.0726102871170291, "rougeL_precision_stderr": 0.0011735561178869995, "rougeL_recall": 0.18252188809890302, "rougeL_recall_stderr": 0.002873571643678364, "rougeLsum_fmeasure": 0.08732410431235793, "rougeLsum_fmeasure_stderr": 0.0014130027217163148, "rougeLsum_precision": 0.061649077224170956, "rougeLsum_precision_stderr": 0.0010164196126269186, "rougeLsum_recall": 0.1564969783302787, "rougeLsum_recall_stderr": 0.002608152410609126}}, "3": {"article_DOC_summary": {"bleu": 0.5312285255112209, "bleu_stderr": 0.055769679108516046, "rouge1_fmeasure": 0.10814433808409338, "rouge1_fmeasure_stderr": 0.0019844919195964802, "rouge1_precision": 0.07901849184085133, "rouge1_precision_stderr": 0.00158149141781139, "rouge1_recall": 0.1876818339260757, "rouge1_recall_stderr": 0.0033845620592438553, "rouge2_fmeasure": 0.015074324129809476, "rouge2_fmeasure_stderr": 0.0008066709490325949, "rouge2_precision": 0.01086933086332831, "rouge2_precision_stderr": 0.0005902700151303653, "rouge2_recall": 0.026805792401099832, "rouge2_recall_stderr": 0.0014700242022978466, "rougeL_fmeasure": 0.09796289395184513, "rougeL_fmeasure_stderr": 0.0017013034910252584, "rougeL_precision": 0.07143623535597288, "rougeL_precision_stderr": 0.0013552097346412838, "rougeL_recall": 0.17061868147380008, "rougeL_recall_stderr": 0.0029548218008992, "rougeLsum_fmeasure": 0.0847515424556008, "rougeLsum_fmeasure_stderr": 0.0015104963660069898, "rougeLsum_precision": 0.06192685725479998, "rougeLsum_precision_stderr": 0.0012248997834784666, "rougeLsum_recall": 0.14788395132010507, "rougeLsum_recall_stderr": 0.0026431485952650934}}, "4": {"article_DOC_summary": {"bleu": 0.2171794656385788, "bleu_stderr": 0.0580106361575993, "rouge1_fmeasure": 0.03205575506161488, "rouge1_fmeasure_stderr": 0.001848822780502997, "rouge1_precision": 0.028569197114255446, "rouge1_precision_stderr": 0.0021093926269611083, "rouge1_recall": 0.049986180880828145, "rouge1_recall_stderr": 0.0029038586809606247, "rouge2_fmeasure": 0.004447159394007513, "rouge2_fmeasure_stderr": 0.0004831807517365398, "rouge2_precision": 0.003992138336784255, "rouge2_precision_stderr": 0.000625486280928393, "rouge2_recall": 0.007143856468458586, "rouge2_recall_stderr": 0.000775527207174529, "rougeL_fmeasure": 0.02857170837956627, "rougeL_fmeasure_stderr": 0.0016148473802429979, "rougeL_precision": 0.02507402908479647, "rougeL_precision_stderr": 0.0018121877476027467, "rougeL_recall": 0.04502723528235586, "rougeL_recall_stderr": 0.0025970015985154675, "rougeLsum_fmeasure": 0.02532162552531629, "rougeLsum_fmeasure_stderr": 0.001460571687341891, "rougeLsum_precision": 0.02280092098484319, "rougeLsum_precision_stderr": 0.0017633844651269714, "rougeLsum_recall": 0.039318241724393174, "rougeLsum_recall_stderr": 0.0022687079801898538}}, "5": {"article_DOC_summary": {"bleu": 1.2989205700146707e-37, "bleu_stderr": 1.6500888245792048e-32, "rouge1_fmeasure": 0.0022764186005474617, "rouge1_fmeasure_stderr": 0.0006298696283878379, "rouge1_precision": 0.0025571229514947014, "rouge1_precision_stderr": 0.0007281415062359935, "rouge1_recall": 0.0021566578919806367, "rouge1_recall_stderr": 0.0005911769409561811, "rouge2_fmeasure": 0.0001281030666243269, "rouge2_fmeasure_stderr": 7.404161859751549e-05, "rouge2_precision": 0.00014412997903563943, "rouge2_precision_stderr": 8.348859628883554e-05, "rouge2_recall": 0.00011864996969267079, "rouge2_recall_stderr": 6.932933515769192e-05, "rougeL_fmeasure": 0.0016885822290001135, "rougeL_fmeasure_stderr": 0.0004582016072242166, "rougeL_precision": 0.0018849622542887335, "rougeL_precision_stderr": 0.000531970687877808, "rougeL_recall": 0.001615365856940405, "rougeL_recall_stderr": 0.0004314744133837763, "rougeLsum_fmeasure": 0.0018318538559767147, "rougeLsum_fmeasure_stderr": 0.0005037674635893385, "rougeLsum_precision": 0.001995793577880732, "rougeLsum_precision_stderr": 0.0005540133520309812, "rougeLsum_recall": 0.0017756009214037463, "rougeLsum_recall_stderr": 0.000488425634026763}}}} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..b234f9148f92ec9899b6b31b5ba3d54bf00d94d8 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.22551871811397353, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02136035868888881 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.05258955980824922, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001111043010483631 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.24440401749751364, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004069008614986987 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.08179555310979232, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0015855832653346308 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.023193335272543247, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006829041826176187 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.11105971904247877, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0028341010971909687 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.03650350909082802, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010163354215107154 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.051210428458407814, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0010759000779014146 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.23873249693202636, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004010114948451509 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.0797122178196054, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015418999962278706 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.050443394209883134, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0010665847609739637 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.23354882611756766, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0038288448419567423 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.07835236496419286, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015149830080257896 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..fc638f639c91787bdb12b67681e96448c4a33853 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.10303247472010832, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.010542121107444794 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.041949850500467546, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001009545354318454 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.21524248765770787, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003637995924121415 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.06582659671546148, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014010035101277287 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.016602402097909204, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0005784273046205441 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.0786634484231439, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002376415534094953 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.025933814076285186, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.000854986393522763 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.041053525063470375, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0009905003168813127 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.21058233480126393, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0035581210952122444 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.0644436752805007, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001375438208183654 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.040395747717907556, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0009891799118591429 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.20549607203927084, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0034995349438813944 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.06327032653279559, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013696218498744507 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..3219903a5a7c161acd0ef8735803068016bc8b26 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.10044377394916508, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.011004903600055578 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.04071859925486269, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.000989754132711725 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2071041537148272, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0036456928364795613 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.06380975510656396, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0013921200872663029 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.016062068279265852, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0005807608561522497 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.07481502747530522, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002374523180072443 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.024990803032608512, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0008536543867796345 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.03972537172680168, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0009640633401932609 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.20198143435079377, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0035579835868813966 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.06228298370655369, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013594322207689349 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.039139482148406846, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0009678826987632891 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.19661295375267737, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0034609252034809553 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.06120828937572442, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013588698800721009 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..6b40192b890db9e985643cf2d80f636f9dc4d50a --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.11695012147806128, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.009564035174544912 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.040613137406636475, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.000993472498566677 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.20769491305475654, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.003694525077541037 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.06398897669967853, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014297013856788472 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.016119706774984935, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0005886522490528559 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.07411626939385718, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0022817865110913976 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.0250870194649846, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0008636090708303351 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.03934321475895627, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0009617791254833742 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2011219485070139, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0035514042056288375 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.06200730116971786, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013847498199754179 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.03873209326674636, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0009654074102020647 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.196124698805037, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0034935507768228203 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.060903726984725515, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001384286630928201 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..133e3cfd94ec0b22849e316ac43acd26f6db9b14 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.09979909326915241, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.015027652350137241 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.040106535729599324, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0011179436243060426 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.20159596944281089, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0036122061036167187 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.06234392347539004, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0013928822038245677 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.01571291807548398, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006433797121782759 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.07190463157595857, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002204303662857119 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.02397153359026078, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0008292654015140437 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.03870233475669446, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0010588745067226968 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.19464293645514244, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003469699527046509 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.06021724808437657, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013459098336822978 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.038217340553490825, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001077208240113436 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.1902899184496612, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.00339164585372067 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.05928914046189273, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013399615128428988 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..28c2661ebf92710880257a5d76c3bf43a960c84a --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.11726441892112459, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.015334082664115558 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.04043384650335457, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0011680539728058876 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.20077044636212277, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0037717120189089287 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.06251485774926024, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014333674940962343 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.016019666538721798, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006963432416328132 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.0727256324755699, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.002280829272562891 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.024261554557884868, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0008527386596796517 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.03899756530778425, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011238312786531195 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.1937695228837636, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.003629063622025122 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.060317175460154866, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001386551318301422 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.03848740805974913, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011314090702100065 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.18913455692813624, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0035362816108656862 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.05935190100165203, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013786302612462235 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..7b3b64d022f13ef0bbb7d9c6342e8979cc1f1f86 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.08326649494563637, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014667171370064546 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.13294991530121725, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0020647880464053562 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.09387610190938273, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014449391906629413 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.008186684744741895, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00038085725309630776 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.014292196596118667, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0007076897173253386 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.009443311010258953, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00042033592017684524 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.0745208321321628, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001231885685841352 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.12093384222921329, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0018396897681209818 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0844430674301303, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001222316625035649 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.07841758933682781, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001356652627657671 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.12608243017908285, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.001944196318168353 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.0886261141852728, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001340925411956822 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.3966713021013514, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.032107788039151756 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..277151d8e58e9784de6242f2a68486d83e6f6bb4 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.08502557604828932, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001409901535332923 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.13428782095164568, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0019616613248484315 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.09647757726185396, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001404507794977568 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.006966193817250453, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00036778884322042845 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.011375012445045086, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.000676347637821329 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.007807306910940247, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00039567014414648163 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.06912434561132932, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001024340907960514 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.11205317271963983, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0015505210767611621 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.07911103532517645, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010225471911468663 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.07928679229201335, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012924928074258797 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.1262165012516492, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0018367956599135702 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.09021046038046152, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0012921693055157244 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.4340075986972079, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03441283754851664 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a44a9acb56ef008d437c42847301de41103df699 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.09291365592791448, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001460793706858565 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.1488422559797642, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0020705710639246084 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.10613644765247322, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001474016161663567 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.009120310744751918, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00039466417821083983 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.015568763230043758, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0007415274403876562 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.01060360006228834, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0004521212932223968 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07481084575169951, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.00106226725810403 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.12310873652544924, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0016574231918608155 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.08624153647417561, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010877894099860167 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.0867915790156753, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013479240543794895 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.1397393265846199, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0019277451004408135 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.09931624380922659, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0013624277073018434 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.5714321541557378, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05207006664300423 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ac4c2b94ae916f0e8e90520b6ec71b672c2e6bef --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.08884062689609729, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001694039009927059 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.13695639134668142, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002297222277021766 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.09749330991596093, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0015999474421077564 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.010317627380981088, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0004915667777414832 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.015891105747890096, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.000746681384466998 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.01117408015693819, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0004880353592289108 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.07195531593720278, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012912975208910194 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.11407921894991604, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0018782347124531222 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.07970227332043539, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0012263741122186553 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.08234443508131861, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0015607302462157188 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.12783583272388502, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002147908605548973 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.09049100197741863, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0014684127049705313 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.6781370863046939, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03589538326068359 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..dcb24c486ab656c66e1067382a055e048efc5628 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.03178506199834185, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001343108102535887 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.04796215079248023, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0018554619120391019 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.03363544745062649, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0012909131114665494 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.003845960090916275, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00032358530255716713 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.006262417812252477, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0005419113367376851 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0042011360449053375, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0003274215003265904 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.026681512204475434, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011001052380045396 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.041081679608834486, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0015755732782817113 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.028264152864206495, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010441583660293569 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.02934075974862999, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012418826658471013 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.04429278174325575, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0017157301035406875 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.03091556784129967, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0011741644318024522 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.1536072771190509, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.01767754971066294 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..4ef5e4a48443febcb511a0ea568d5bccab7359db --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.004982978359031628, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0005510614976068719 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.008004146641079459, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.000886422861824466 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.005186538590453919, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0005298157893824338 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.0005839050846852418, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00012587591454986364 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0013034289489588643, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.00039703429859922704 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0006129505049971475, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00012136340321994513 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.004267497431002324, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0004621334041194036 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.0070849664096893075, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0007922992743738505 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.004489388399312842, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0004482804717777068 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.004528845080105699, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0005065107975618541 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.007377839412377322, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0008304090072360563 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.0047159872335650254, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0004824536058253248 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.6675204478839374e-07, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 3.1908739777991254e-07 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..72b93460eadff04945e48d8318097a43f7b0a1e8 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 0.0, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.0 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.0001111111111111111, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.00011111111111110651 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 5.1282051282051286e-05, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 5.128205128205036e-05 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 7.017543859649122e-05, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 7.01754385964955e-05 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.0, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.0, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.0, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.0001111111111111111, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.00011111111111110651 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 5.1282051282051286e-05, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 5.128205128205036e-05 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 7.017543859649122e-05, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 7.01754385964955e-05 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.0001111111111111111, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.00011111111111110651 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 5.1282051282051286e-05, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 5.128205128205036e-05 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 7.017543859649122e-05, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 7.01754385964955e-05 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..74bc058096ce7d2fb441c9999a2d6c02d165e139 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 2.505657804731795, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.08448129330294266 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.2842523517699498, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0036605089796832337 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.21222024181682922, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002811877975435317 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.22515985559604443, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.00271913604007141 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.07600913474511695, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002249723324011366 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.053107638913185144, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0012617608610165124 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.05651165965956718, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0012897141937349183 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.2157915985634754, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0030386856144986054 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.15690291192122693, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002118913610738115 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.16712068957307108, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0020525400121234427 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.23794716782068137, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0032984804753267384 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.1745112707430124, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0023783566030993763 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.1858300404558237, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002325463614772228 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4a84330be6fab580a32d7a9ec4b94ad6596ab144 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 1.4491558022298123, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.11677284059045918 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.2201653187301119, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.004149940282223749 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1568229775570194, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0029810729084815647 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.1661375380722948, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0029609391315106206 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.06340195645403277, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002578554366386417 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.03935514148515296, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0011857326939533532 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.04173045009950831, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0012076024017352199 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.1713871764712513, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0034976614480037205 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.11716713627352698, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022315452547388387 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.12476790715224081, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0022207952377841602 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.1863421045122678, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0037210079404938544 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.12949037161998142, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025070704307576055 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.13765149537351365, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0024972853701660013 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..33ae54bc97d0e0c6773bb8178f2c0ddbf8509152 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 1.0325170083670192, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.09155070159493152 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.20550875384522863, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0043850189033791806 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1326798020968605, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002874666802950118 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.1441917973906388, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002913542035143854 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.06261701191581304, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002785773123682906 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.03406135061551867, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0011419084537411385 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.036857023400034224, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0011646994165157424 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.16608649290312155, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.003830938059547555 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.10219810669843173, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002215580186273407 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.11164498037115686, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0022462708577540924 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.17865049094772503, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.004013254974233051 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.11190357275012455, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002444788908100573 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.12207928897340385, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0024841685357783203 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c38493d43aa43984189333cfe9205e9f0010c0b1 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 1.134167256140814, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.09717606546886882 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.22909638360887752, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.004632029151207689 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1376768668142333, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002850489300883519 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.1525785492028794, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002923087860358927 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.07440792663345197, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0032019898187815683 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.0363659394734723, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0011645151702284 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.0401771744546227, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0012133220144334815 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.1883176484638121, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0041502761353747275 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.10660833450522245, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022044580465938987 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.11918515988014529, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0022752870246164156 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.2012193449008338, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.004320199440964196 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.11630225750788287, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0024288025364187717 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.129729548382001, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002508780371362807 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..be439fb6d3fbaf70621e65a2eb71751318fd42a4 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 1.1360525639294996, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.0860200220653462 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.23672530803742303, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.004759457434672915 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1412752859993294, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002871463092526642 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.15733151178469185, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0029744474367067995 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.0807897134779241, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0034137441942331476 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.036442980899354716, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0011296048388216163 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.04108775721036089, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0012181280627287497 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.19449993295677506, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.004259405751648249 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.10963957627610135, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002205873184274684 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.12310488343683532, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.002309424803007831 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.20742302690599165, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.004423367405689027 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.11944436234394012, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002441502584100248 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.13370390032101528, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.00254631083155397 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_0.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..5b0de68bed397a82cc71ed9dd495931f1f88415b --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.083586610509537, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.001440762933833016 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2070947923042907, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0032558812336387854 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.11685859723410175, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0018776410036142014 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.011753995351515174, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0005738763631080483 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.030656799939853128, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0015084197234712328 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.01676685311987567, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0008116939067346654 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07650333684239294, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0012273067013688272 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.1906362003380271, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0028813349712405664 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10717031246925551, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016214726366478817 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.06525567378934424, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0011000354258816365 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.16315360866507916, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0026072800176013687 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.09135836009153098, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0014400202436564515 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.5153278620265127, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.057252202800862795 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_1.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..00efedec0b98ef28a86b6f5aba644bdbf87f2989 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.08126923644752823, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0013883046856172049 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.20202344787769308, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003281264570631814 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.11434185937890248, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0018809140186798994 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.011454919678657324, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0005726609882831751 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.029865905993707347, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0015330029947613294 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.01631323028144093, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0008130028904452307 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07392178960902045, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0012059856726637796 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.18437150774997185, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0029021081747433526 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10408841638064358, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016358693720022312 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.06395968595013059, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001077632692936246 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.16048243217413274, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0026288626085393357 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.09016934798656069, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.001463003122509554 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.5157175151500965, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.06374111661980116 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_2.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a1d6498e412839dd0a7fe733127d1ce4d3c507d7 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0796828677477615, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0013416709097367066 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.19965767912316895, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003229270721434028 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.11248136961296866, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0018348661538257311 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.01056816485435896, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0005450902704398356 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.02805644943459805, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0014983600532259245 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.015174398982269022, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0007849834892871412 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0726102871170291, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0011735561178869995 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.18252188809890302, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.002873571643678364 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10258401216745326, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.001610972598460782 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.061649077224170956, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0010164196126269186 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.1564969783302787, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.002608152410609126 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.08732410431235793, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0014130027217163148 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.46599495916506006, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.049893191051982456 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_3.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b8058c2af21fca659409fe2eabab38712888e56a --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.07901849184085133, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.00158149141781139 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.1876818339260757, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0033845620592438553 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.10814433808409338, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0019844919195964802 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.01086933086332831, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0005902700151303653 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.026805792401099832, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0014700242022978466 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.015074324129809476, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0008066709490325949 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07143623535597288, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0013552097346412838 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.17061868147380008, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0029548218008992 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.09796289395184513, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0017013034910252584 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.06192685725479998, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0012248997834784666 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.14788395132010507, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0026431485952650934 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0847515424556008, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0015104963660069898 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.5312285255112209, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.055769679108516046 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_4.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ec54b939ed52136ab1d25b917edc8356e74741f2 --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.028569197114255446, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0021093926269611083 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.049986180880828145, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0029038586809606247 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.03205575506161488, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.001848822780502997 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.003992138336784255, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.000625486280928393 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.007143856468458586, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.000775527207174529 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.004447159394007513, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0004831807517365398 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.02507402908479647, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0018121877476027467 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.04502723528235586, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0025970015985154675 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.02857170837956627, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016148473802429979 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.02280092098484319, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0017633844651269714 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.039318241724393174, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0022687079801898538 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.02532162552531629, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.001460571687341891 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.2171794656385788, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.0580106361575993 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_5.json b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..8afbafa6cda39e5c77c342b782691648f507689f --- /dev/null +++ b/1b11b51b5/evaluation/generation/slim.1b11b51b5_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0025571229514947014, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0007281415062359935 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.0021566578919806367, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0005911769409561811 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.0022764186005474617, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0006298696283878379 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.00014412997903563943, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 8.348859628883554e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.00011864996969267079, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 6.932933515769192e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0001281030666243269, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 7.404161859751549e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0018849622542887335, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.000531970687877808 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.001615365856940405, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0004314744133837763 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.0016885822290001135, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0004582016072242166 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.001995793577880732, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0005540133520309812 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.0017756009214037463, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.000488425634026763 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0018318538559767147, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0005037674635893385 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.2989205700146707e-37, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 1.6500888245792048e-32 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/1b11b51b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_0.csv b/1b11b51b5/evaluation/rankeval/1b11b51b5_0.csv new file mode 100644 index 0000000000000000000000000000000000000000..8c88cd76fc06b91cd23900fb29d83ac36616cfdf --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_0.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.341,0.014998131348402704,0 +anli_r2,acc,0.33,0.014876872027456734,0 +anli_r3,acc,0.3325,0.013605417345710528,0 +arc_challenge,acc,0.1766211604095563,0.01114404276931652,0 +arc_challenge,acc_norm,0.2150170648464164,0.012005717634133611,0 +arc_easy,acc,0.35858585858585856,0.009840882301225297,0 +arc_easy,acc_norm,0.32954545454545453,0.009645184190953861,0 +boolq,acc,0.6134556574923548,0.00851694393434197,1 +cb,acc,0.32142857142857145,0.06297362289056341,1 +cb,f1,0.16666666666666666,,1 +copa,acc,0.53,0.05016135580465919,0 +hellaswag,acc,0.2689703246365266,0.004425182676353208,0 +hellaswag,acc_norm,0.27394941246763593,0.00445071867355267,0 +piqa,acc,0.5941240478781284,0.011457256809261785,0 +piqa,acc_norm,0.5859630032644179,0.011492118481417578,0 +rte,acc,0.51985559566787,0.030072723167317177,0 +sciq,acc,0.543,0.01576069159013638,0 +sciq,acc_norm,0.496,0.01581879370351089,0 +storycloze_2016,acc,0.5537145911277391,0.011495517440721685,0 +winogrande,acc,0.5138121546961326,0.014047122916440412,0 diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_0.json b/1b11b51b5/evaluation/rankeval/1b11b51b5_0.json new file mode 100644 index 0000000000000000000000000000000000000000..28c09282bdd81f8499363d295578f2cdf9167ee9 --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.341, + "acc_stderr": 0.014998131348402704 + }, + "anli_r2": { + "acc": 0.33, + "acc_stderr": 0.014876872027456734 + }, + "anli_r3": { + "acc": 0.3325, + "acc_stderr": 0.013605417345710528 + }, + "cb": { + "acc": 0.32142857142857145, + "acc_stderr": 0.06297362289056341, + "f1": 0.16666666666666666 + }, + "copa": { + "acc": 0.53, + "acc_stderr": 0.05016135580465919 + }, + "hellaswag": { + "acc": 0.2689703246365266, + "acc_stderr": 0.004425182676353208, + "acc_norm": 0.27394941246763593, + "acc_norm_stderr": 0.00445071867355267 + }, + "rte": { + "acc": 0.51985559566787, + "acc_stderr": 0.030072723167317177 + }, + "winogrande": { + "acc": 0.5138121546961326, + "acc_stderr": 0.014047122916440412 + }, + "storycloze_2016": { + "acc": 0.5537145911277391, + "acc_stderr": 0.011495517440721685 + }, + "boolq": { + "acc": 0.6134556574923548, + "acc_stderr": 0.00851694393434197 + }, + "arc_easy": { + "acc": 0.35858585858585856, + "acc_stderr": 0.009840882301225297, + "acc_norm": 0.32954545454545453, + "acc_norm_stderr": 0.009645184190953861 + }, + "arc_challenge": { + "acc": 0.1766211604095563, + "acc_stderr": 0.01114404276931652, + "acc_norm": 0.2150170648464164, + "acc_norm_stderr": 0.012005717634133611 + }, + "sciq": { + "acc": 0.543, + "acc_stderr": 0.01576069159013638, + "acc_norm": 0.496, + "acc_norm_stderr": 0.01581879370351089 + }, + "piqa": { + "acc": 0.5941240478781284, + "acc_stderr": 0.011457256809261785, + "acc_norm": 0.5859630032644179, + "acc_norm_stderr": 0.011492118481417578 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_1.csv b/1b11b51b5/evaluation/rankeval/1b11b51b5_1.csv new file mode 100644 index 0000000000000000000000000000000000000000..3809f3d92532892cbbe519939ff6978671139ac3 --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_1.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.349,0.015080663991563098,0 +anli_r2,acc,0.329,0.014865395385928357,0 +anli_r3,acc,0.3258333333333333,0.013535422043417455,0 +arc_challenge,acc,0.181740614334471,0.011269198948880236,0 +arc_challenge,acc_norm,0.2150170648464164,0.01200571763413361,0 +arc_easy,acc,0.35185185185185186,0.009799078929868706,0 +arc_easy,acc_norm,0.32365319865319864,0.009600478182273778,0 +boolq,acc,0.5211009174311927,0.00873726405694862,1 +cb,acc,0.5,0.06741998624632421,1 +cb,f1,0.3261261261261261,,1 +copa,acc,0.57,0.04975698519562428,0 +hellaswag,acc,0.2682732523401713,0.004421551307678477,0 +hellaswag,acc_norm,0.27165903206532566,0.004439059440526252,0 +piqa,acc,0.5952121871599565,0.011452361375057032,0 +piqa,acc_norm,0.5848748639825898,0.011496520442659124,0 +rte,acc,0.5379061371841155,0.030009848912529113,0 +sciq,acc,0.548,0.015746235865880677,0 +sciq,acc_norm,0.508,0.015817274929209015,0 +storycloze_2016,acc,0.5440940673436665,0.011517383123961531,0 +winogrande,acc,0.4972375690607735,0.014052271211616448,0 diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_1.json b/1b11b51b5/evaluation/rankeval/1b11b51b5_1.json new file mode 100644 index 0000000000000000000000000000000000000000..dd3bce1b102d0cf95b4905bea4fb6c4a36256f88 --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.349, + "acc_stderr": 0.015080663991563098 + }, + "anli_r2": { + "acc": 0.329, + "acc_stderr": 0.014865395385928357 + }, + "anli_r3": { + "acc": 0.3258333333333333, + "acc_stderr": 0.013535422043417455 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.3261261261261261 + }, + "copa": { + "acc": 0.57, + "acc_stderr": 0.04975698519562428 + }, + "hellaswag": { + "acc": 0.2682732523401713, + "acc_stderr": 0.004421551307678477, + "acc_norm": 0.27165903206532566, + "acc_norm_stderr": 0.004439059440526252 + }, + "rte": { + "acc": 0.5379061371841155, + "acc_stderr": 0.030009848912529113 + }, + "winogrande": { + "acc": 0.4972375690607735, + "acc_stderr": 0.014052271211616448 + }, + "storycloze_2016": { + "acc": 0.5440940673436665, + "acc_stderr": 0.011517383123961531 + }, + "boolq": { + "acc": 0.5211009174311927, + "acc_stderr": 0.00873726405694862 + }, + "arc_easy": { + "acc": 0.35185185185185186, + "acc_stderr": 0.009799078929868706, + "acc_norm": 0.32365319865319864, + "acc_norm_stderr": 0.009600478182273778 + }, + "arc_challenge": { + "acc": 0.181740614334471, + "acc_stderr": 0.011269198948880236, + "acc_norm": 0.2150170648464164, + "acc_norm_stderr": 0.01200571763413361 + }, + "sciq": { + "acc": 0.548, + "acc_stderr": 0.015746235865880677, + "acc_norm": 0.508, + "acc_norm_stderr": 0.015817274929209015 + }, + "piqa": { + "acc": 0.5952121871599565, + "acc_stderr": 0.011452361375057032, + "acc_norm": 0.5848748639825898, + "acc_norm_stderr": 0.011496520442659124 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_2.csv b/1b11b51b5/evaluation/rankeval/1b11b51b5_2.csv new file mode 100644 index 0000000000000000000000000000000000000000..ca0f4716fee4afccdcde988e9258df11fd9d71c5 --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_2.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.323,0.014794927843348635,0 +anli_r2,acc,0.338,0.014965960710224487,0 +anli_r3,acc,0.3441666666666667,0.013720551062295756,0 +arc_challenge,acc,0.17320819112627986,0.011058694183280326,0 +arc_challenge,acc_norm,0.2090443686006826,0.011882746987406446,0 +arc_easy,acc,0.343013468013468,0.00974096566648923,0 +arc_easy,acc_norm,0.3261784511784512,0.009619849417035161,0 +boolq,acc,0.5351681957186545,0.008723396352960188,1 +cb,acc,0.4107142857142857,0.0663363415035954,1 +cb,f1,0.2283333333333333,,1 +copa,acc,0.54,0.05009082659620332,0 +hellaswag,acc,0.26797450707030473,0.00441999074191598,0 +hellaswag,acc_norm,0.2727544313881697,0.004444654076550554,0 +piqa,acc,0.5930359085963003,0.011462093919190166,0 +piqa,acc_norm,0.5859630032644179,0.011492118481417578,0 +rte,acc,0.5126353790613718,0.030086851767188564,0 +sciq,acc,0.54,0.015768596914394386,0 +sciq,acc_norm,0.505,0.01581850894443666,0 +storycloze_2016,acc,0.5446285408872261,0.011516282203726658,0 +winogrande,acc,0.5027624309392266,0.014052271211616436,0 diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_2.json b/1b11b51b5/evaluation/rankeval/1b11b51b5_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ff536a36fd2c8ede311ef9afd34f58ebb69f5033 --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.323, + "acc_stderr": 0.014794927843348635 + }, + "anli_r2": { + "acc": 0.338, + "acc_stderr": 0.014965960710224487 + }, + "anli_r3": { + "acc": 0.3441666666666667, + "acc_stderr": 0.013720551062295756 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.2283333333333333 + }, + "copa": { + "acc": 0.54, + "acc_stderr": 0.05009082659620332 + }, + "hellaswag": { + "acc": 0.26797450707030473, + "acc_stderr": 0.00441999074191598, + "acc_norm": 0.2727544313881697, + "acc_norm_stderr": 0.004444654076550554 + }, + "rte": { + "acc": 0.5126353790613718, + "acc_stderr": 0.030086851767188564 + }, + "winogrande": { + "acc": 0.5027624309392266, + "acc_stderr": 0.014052271211616436 + }, + "storycloze_2016": { + "acc": 0.5446285408872261, + "acc_stderr": 0.011516282203726658 + }, + "boolq": { + "acc": 0.5351681957186545, + "acc_stderr": 0.008723396352960188 + }, + "arc_easy": { + "acc": 0.343013468013468, + "acc_stderr": 0.00974096566648923, + "acc_norm": 0.3261784511784512, + "acc_norm_stderr": 0.009619849417035161 + }, + "arc_challenge": { + "acc": 0.17320819112627986, + "acc_stderr": 0.011058694183280326, + "acc_norm": 0.2090443686006826, + "acc_norm_stderr": 0.011882746987406446 + }, + "sciq": { + "acc": 0.54, + "acc_stderr": 0.015768596914394386, + "acc_norm": 0.505, + "acc_norm_stderr": 0.01581850894443666 + }, + "piqa": { + "acc": 0.5930359085963003, + "acc_stderr": 0.011462093919190166, + "acc_norm": 0.5859630032644179, + "acc_norm_stderr": 0.011492118481417578 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_3.csv b/1b11b51b5/evaluation/rankeval/1b11b51b5_3.csv new file mode 100644 index 0000000000000000000000000000000000000000..b400b6c2eb6a155b007d31f617656eb90d0e173e --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_3.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.337,0.014955087918653595,0 +anli_r2,acc,0.323,0.01479492784334864,0 +anli_r3,acc,0.3283333333333333,0.013562032919529026,0 +arc_challenge,acc,0.19368600682593856,0.01154842540997854,0 +arc_challenge,acc_norm,0.21928327645051193,0.01209124578761574,0 +arc_easy,acc,0.351010101010101,0.009793703885101047,0 +arc_easy,acc_norm,0.335016835016835,0.009685160765932357,0 +boolq,acc,0.5400611620795107,0.008716939962184103,1 +cb,acc,0.42857142857142855,0.06672848092813058,1 +cb,f1,0.23599320882852293,,1 +copa,acc,0.59,0.049431107042371025,0 +hellaswag,acc,0.26996614220274845,0.00443034623465038,0 +hellaswag,acc_norm,0.2741485759808803,0.00445172553062638,0 +piqa,acc,0.6011969532100109,0.011424390545037282,0 +piqa,acc_norm,0.5832426550598476,0.011503015163618313,0 +rte,acc,0.5379061371841155,0.030009848912529117,0 +sciq,acc,0.542,0.015763390640483703,0 +sciq,acc_norm,0.51,0.015816135752773214,0 +storycloze_2016,acc,0.5451630144307856,0.011515167912227987,0 +winogrande,acc,0.5011838989739542,0.014052446290529015,0 diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_3.json b/1b11b51b5/evaluation/rankeval/1b11b51b5_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a88079726704072dd5a950e37edeca498d1ef0a2 --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.337, + "acc_stderr": 0.014955087918653595 + }, + "anli_r2": { + "acc": 0.323, + "acc_stderr": 0.01479492784334864 + }, + "anli_r3": { + "acc": 0.3283333333333333, + "acc_stderr": 0.013562032919529026 + }, + "cb": { + "acc": 0.42857142857142855, + "acc_stderr": 0.06672848092813058, + "f1": 0.23599320882852293 + }, + "copa": { + "acc": 0.59, + "acc_stderr": 0.049431107042371025 + }, + "hellaswag": { + "acc": 0.26996614220274845, + "acc_stderr": 0.00443034623465038, + "acc_norm": 0.2741485759808803, + "acc_norm_stderr": 0.00445172553062638 + }, + "rte": { + "acc": 0.5379061371841155, + "acc_stderr": 0.030009848912529117 + }, + "winogrande": { + "acc": 0.5011838989739542, + "acc_stderr": 0.014052446290529015 + }, + "storycloze_2016": { + "acc": 0.5451630144307856, + "acc_stderr": 0.011515167912227987 + }, + "boolq": { + "acc": 0.5400611620795107, + "acc_stderr": 0.008716939962184103 + }, + "arc_easy": { + "acc": 0.351010101010101, + "acc_stderr": 0.009793703885101047, + "acc_norm": 0.335016835016835, + "acc_norm_stderr": 0.009685160765932357 + }, + "arc_challenge": { + "acc": 0.19368600682593856, + "acc_stderr": 0.01154842540997854, + "acc_norm": 0.21928327645051193, + "acc_norm_stderr": 0.01209124578761574 + }, + "sciq": { + "acc": 0.542, + "acc_stderr": 0.015763390640483703, + "acc_norm": 0.51, + "acc_norm_stderr": 0.015816135752773214 + }, + "piqa": { + "acc": 0.6011969532100109, + "acc_stderr": 0.011424390545037282, + "acc_norm": 0.5832426550598476, + "acc_norm_stderr": 0.011503015163618313 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_4.csv b/1b11b51b5/evaluation/rankeval/1b11b51b5_4.csv new file mode 100644 index 0000000000000000000000000000000000000000..d2fefd3fcdf1b56205b62699a4a23c7424a29a1a --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_4.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.349,0.015080663991563097,0 +anli_r2,acc,0.336,0.014944140233795028,0 +anli_r3,acc,0.3358333333333333,0.01363926119093288,0 +arc_challenge,acc,0.18344709897610922,0.011310170179554536,0 +arc_challenge,acc_norm,0.21843003412969283,0.012074291605700994,0 +arc_easy,acc,0.33585858585858586,0.009691180932083505,0 +arc_easy,acc_norm,0.32575757575757575,0.009616642976885964,0 +boolq,acc,0.5464831804281346,0.008707182331111646,1 +cb,acc,0.42857142857142855,0.06672848092813057,1 +cb,f1,0.23599320882852293,,1 +copa,acc,0.58,0.049604496374885836,0 +hellaswag,acc,0.2688707428799044,0.004424664761480205,0 +hellaswag,acc_norm,0.2726548496315475,0.004444146875436291,0 +piqa,acc,0.5941240478781284,0.011457256809261783,0 +piqa,acc_norm,0.5859630032644179,0.011492118481417578,0 +rte,acc,0.516245487364621,0.030080573208738064,0 +sciq,acc,0.531,0.01578886595953901,0 +sciq,acc_norm,0.526,0.015797897758042755,0 +storycloze_2016,acc,0.549973276322822,0.011504536925414017,0 +winogrande,acc,0.505130228887135,0.014051745961790516,0 diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_4.json b/1b11b51b5/evaluation/rankeval/1b11b51b5_4.json new file mode 100644 index 0000000000000000000000000000000000000000..afcd5e2cfec1266548b7eb3037a8e3510313941f --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_4.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.349, + "acc_stderr": 0.015080663991563097 + }, + "anli_r2": { + "acc": 0.336, + "acc_stderr": 0.014944140233795028 + }, + "anli_r3": { + "acc": 0.3358333333333333, + "acc_stderr": 0.01363926119093288 + }, + "cb": { + "acc": 0.42857142857142855, + "acc_stderr": 0.06672848092813057, + "f1": 0.23599320882852293 + }, + "copa": { + "acc": 0.58, + "acc_stderr": 0.049604496374885836 + }, + "hellaswag": { + "acc": 0.2688707428799044, + "acc_stderr": 0.004424664761480205, + "acc_norm": 0.2726548496315475, + "acc_norm_stderr": 0.004444146875436291 + }, + "rte": { + "acc": 0.516245487364621, + "acc_stderr": 0.030080573208738064 + }, + "winogrande": { + "acc": 0.505130228887135, + "acc_stderr": 0.014051745961790516 + }, + "storycloze_2016": { + "acc": 0.549973276322822, + "acc_stderr": 0.011504536925414017 + }, + "boolq": { + "acc": 0.5464831804281346, + "acc_stderr": 0.008707182331111646 + }, + "arc_easy": { + "acc": 0.33585858585858586, + "acc_stderr": 0.009691180932083505, + "acc_norm": 0.32575757575757575, + "acc_norm_stderr": 0.009616642976885964 + }, + "arc_challenge": { + "acc": 0.18344709897610922, + "acc_stderr": 0.011310170179554536, + "acc_norm": 0.21843003412969283, + "acc_norm_stderr": 0.012074291605700994 + }, + "sciq": { + "acc": 0.531, + "acc_stderr": 0.01578886595953901, + "acc_norm": 0.526, + "acc_norm_stderr": 0.015797897758042755 + }, + "piqa": { + "acc": 0.5941240478781284, + "acc_stderr": 0.011457256809261783, + "acc_norm": 0.5859630032644179, + "acc_norm_stderr": 0.011492118481417578 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_5.csv b/1b11b51b5/evaluation/rankeval/1b11b51b5_5.csv new file mode 100644 index 0000000000000000000000000000000000000000..c39e1a797fefc1e377e55558b1949ff1382242ec --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_5.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.333,0.01491084616422986,0 +anli_r2,acc,0.328,0.014853842487270336,0 +anli_r3,acc,0.3333333333333333,0.0136139500102256,0 +arc_challenge,acc,0.1689419795221843,0.01094979565248503,0 +arc_challenge,acc_norm,0.2167235494880546,0.01204015671348119,0 +arc_easy,acc,0.3552188552188552,0.00982024589928712,0 +arc_easy,acc_norm,0.3333333333333333,0.00967301666813339,0 +boolq,acc,0.5376146788990825,0.00872027373643368,1 +cb,acc,0.39285714285714285,0.0658538889806635,1 +cb,f1,0.18803418803418803,,1 +copa,acc,0.55,0.04999999999999999,0 +hellaswag,acc,0.26996614220274845,0.00443034623465038,0 +hellaswag,acc_norm,0.2740489942242581,0.004451222241494057,0 +piqa,acc,0.5941240478781284,0.011457256809261785,0 +piqa,acc_norm,0.5788900979325353,0.011519701059151495,0 +rte,acc,0.51985559566787,0.030072723167317184,0 +sciq,acc,0.541,0.01576602573788216,0 +sciq,acc_norm,0.506,0.015818160898606715,0 +storycloze_2016,acc,0.5446285408872261,0.011516282203726658,0 +winogrande,acc,0.5067087608524072,0.014051220692330349,0 diff --git a/1b11b51b5/evaluation/rankeval/1b11b51b5_5.json b/1b11b51b5/evaluation/rankeval/1b11b51b5_5.json new file mode 100644 index 0000000000000000000000000000000000000000..6db8c403847cf9e4cd5899a199284489ae15654e --- /dev/null +++ b/1b11b51b5/evaluation/rankeval/1b11b51b5_5.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.333, + "acc_stderr": 0.01491084616422986 + }, + "anli_r2": { + "acc": 0.328, + "acc_stderr": 0.014853842487270336 + }, + "anli_r3": { + "acc": 0.3333333333333333, + "acc_stderr": 0.0136139500102256 + }, + "cb": { + "acc": 0.39285714285714285, + "acc_stderr": 0.0658538889806635, + "f1": 0.18803418803418803 + }, + "copa": { + "acc": 0.55, + "acc_stderr": 0.04999999999999999 + }, + "hellaswag": { + "acc": 0.26996614220274845, + "acc_stderr": 0.00443034623465038, + "acc_norm": 0.2740489942242581, + "acc_norm_stderr": 0.004451222241494057 + }, + "rte": { + "acc": 0.51985559566787, + "acc_stderr": 0.030072723167317184 + }, + "winogrande": { + "acc": 0.5067087608524072, + "acc_stderr": 0.014051220692330349 + }, + "storycloze_2016": { + "acc": 0.5446285408872261, + "acc_stderr": 0.011516282203726658 + }, + "boolq": { + "acc": 0.5376146788990825, + "acc_stderr": 0.00872027373643368 + }, + "arc_easy": { + "acc": 0.3552188552188552, + "acc_stderr": 0.00982024589928712, + "acc_norm": 0.3333333333333333, + "acc_norm_stderr": 0.00967301666813339 + }, + "arc_challenge": { + "acc": 0.1689419795221843, + "acc_stderr": 0.01094979565248503, + "acc_norm": 0.2167235494880546, + "acc_norm_stderr": 0.01204015671348119 + }, + "sciq": { + "acc": 0.541, + "acc_stderr": 0.01576602573788216, + "acc_norm": 0.506, + "acc_norm_stderr": 0.015818160898606715 + }, + "piqa": { + "acc": 0.5941240478781284, + "acc_stderr": 0.011457256809261785, + "acc_norm": 0.5788900979325353, + "acc_norm_stderr": 0.011519701059151495 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c77099b880bbd372acd4a4b2d536442f647587fb --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f465ab9ca7713817b51d24d92452b32d3579f5b4efcd729618cb536a4a0c66d1 +size 205568023 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82f3f77a07b585227325c2f933e087ae15642343 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193a924b70e2d577f29d47a713520117f52bc8784e0b21764fa09c24f8105eb7 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ad2d956b51a814379677a5d2968baae83907268 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:602c664e2bfef382a6163fb8ba57e4ed30f31c109e4a63fa4a7f74e54ce6c2f3 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5daf084711f373f909cd43cb8781fcf303280377 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f27b219906d5da46b5676bfcef56a5cedbee4424edf7d5d63fa71da055a41a0 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b5dbde71ea150f1088157033c892cbc4a77da44 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:583bcb606d464facedbf6c52b2ad90c79909d95d1628654eaf264d031d78c067 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..638a0698be3077740f162643dfff785d9f255c37 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9252cce199299ed895f7fe324482fbcb0985edfef039ffc516479e243c1b2b91 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f97965eb9118c094a068a72b88bbefc3f82ce74e --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c054d4284056fc8e8bf8a621105e3607a6536d37002f09665db367b67f1e7fad +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ac51de3be71323e5c0603ab05dbebc801f1f179 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7385353acaafe95347f10ea30aea0c68d91ad1a9d46bb479280c14d98d787f7 +size 205567970 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c198a26eb87e2d36dd89e2f175c93d06e319d0d --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96aa648cba2a36cc4d363a059cc1cc9fc4add4ee71174906395ffe1088fcbabe +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86c00ad8149459ae6e9bd90b1e61789d09b83f72 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34219a4dbb7f5de81b847074ad3d77704917188cf194b4739f8613e5caf378ca +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef1c66ea5d8daa47574231cd0bb9e9d521454c1c --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d8916f21e17244e57c0f887c5c0ec7264739c15b79c02cecc382169f99a3ca +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b029a7c478f6f485156156146ca870459406f075 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0431f7b6d244d65f45d3ec191f9e44efc0ff540334bda4aab2949b1a57232e3d +size 205568151 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..280a1cab5ceed186c5dc08e4782cd4c61196acbe --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2822666b7d30b430a70cc50b39d354cdf7dedfacccd32e5134951ff89220c1c6 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..304670089e966edf41ba39f3d7c5ee4a145edca1 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44a1e72fd13b7a5e2cb7540c9ff7cb2a8f7134db1e196c5eca8faf348d901ee3 +size 205568034 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f606d0252088f0ae5eb97de799baf0858ca1eb3f --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf1894bec64ad75651068999eb8fbf6ef74ab0233328b33d3e7f6e4ee21b1a4 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e02369a55b567d417cba445f97eb5686c877f0c1 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88535e1b15c279763250b669bbac8ed225856f5f35c09a476ec9c31fa025200a +size 205568098 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82166b1bd5564c4385e5c985694cc62048f539c5 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21437a6728ba3c51904d50dd299fb6fd4f37a02e589933269933a04087335b9d +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a3415eb3c9905847d73169ca746fdc94a89dd13 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a7ab56faadc47fe3da0c5e8a52630161a4e54444e7040811a4df6ccf5b48f8b +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ff11ad4821fb0765afd18a6ffc8d988d000a558 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd095032890e654d56e7ed8c78ab8694d29c5907b8a65e22da532b7148f0539c +size 205568034 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8eb4ec0cf234d73659bfe8e332c085dbd7bdb7db --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ecad86201a2a6c86280fc2cc89ab3e0b1431051da04fa43ea86f7152a8f23a8 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5845d0cb03106b0523d7bb68d8ee019804c122ae --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e0da892484c33079e4c7bd15a15a19221ea30b0badcdc18d6acd44e5ac83cd4 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18e27eea2b220c9d5dda867043074226c71b6991 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a0479d1c9e56738fbeca30ffe7d44e270cc19c8c27a4c916bf1961d8db2e268 +size 205568098 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c81d25d0c8762885f7352783350d02cd71dc80be --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffe735e099c241e16fe1d186382f692fbc584a05090d6084114e04aac67646d6 +size 205568151 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d744ecf4fd5299c14fbc1d9f2b49a7e789e76ac --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9965d03b9128a5b8636110a20a1f92b087c616ef0c1ff3b465e3d87cfe5a131d +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00fcdb4d86a914839142a22f696766764970c6ce --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42d4378098149f507a5912d0fd8589a0d7ade6732bcd12d43e701fd2c4e070cc +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eaf6f5f7cce43b9faf68da2fdf40d3ccef310eb3 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af2f7be60892f676caf10d77ccb23419f36214935f9973f9ac2de8544c4f74c5 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a642f21a356b3a0bc76bf8930525dcd17266cdc9 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3476364a7c8dea0b03153ba40f35f3569ca4d5e1be4ce009744d98db4364c5b +size 205568034 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d56310303513b3eef09814d401d9aae204939454 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acccd5861086e74d0928fc2d884017c0484ca0b9f75d66a3296dddd890d9af1b +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef1a1b222da63c4a70d5324b88800e8552eab439 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44aa1a0e5d2e2f6f91020d453bfbb4a38f72a6a8b8be0e19c256cfedd09c872c +size 205568290 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c31a939db6199a143f8b2f4d1b1baa38a6d945c7 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:138ed8440308cbbf07417ae92ba949d35870f0e5fbc8476bca0a983758724749 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..daeba79c7b832ab582d4b018313b1e14640a3648 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e616044680a1d2a3209d93506dc97cfe9e7107e8a6d02c0d03513b0216cabfed +size 205568098 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8570e7854783b0581e40a25ce9332bdb58d1054 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f01a589005db58a3412bf11ae2f107ea16c1cd1318815a4652b005a4f39aea +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36cf9335e02621ab5b5f8c5df9764fd5261deb2c --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbe1cf4666ced04e802a7ec0b1dab3a4a66cbec5eaa0d6d3cc2088c57031781b +size 205568098 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d587f634b8b2b470e0037a593af73157523c8131 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42973011f6c15c60a97a7c37cef7a1204d70ecdb95a97d48bfe82a5efdc5bb5a +size 205568215 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75ae4ce3ff2cbac59d0f166ac54328523f01975b --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67b8b603efd992281541a058c31bf9da03b42def7729290a4243f46269812985 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..360dff3728c66a1e987e4b1e0183e788e0b1ae83 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8a158aceeca446da8111c35168fbb0d4ec435bfdbb6fcd86c7fb2f4c13a540 +size 205568354 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71d79df23e1abb74c6982f49e33d3e13ae5b1daa --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a0d7ffc58fa68ee82ce6b340510f3ebbd2c2c417b9750e2f0fc977caa41340c +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..121bff5d514f2eb5ac6585436490b7237c6d51d5 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b5770c96ba3c9ac0dc86fdf7b0c6d0b0f378367b6687c7a7b8a41678639db34 +size 205568034 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fb8c476902a5d9d18def9cc835f57f501a540b7 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:510a2112d6d813cc3880649043dab1824a71e8f6ce47b4c9b99335ed04f3f3bd +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9363905535fabf86f88c40e09b1fcb18f1890229 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2518b638971a2e715720a0a308aa998d5a852401615b3a58fcb339f814ad8e3f +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a347b1b3d66e09f03704ab2594bb527be22742cb --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9486c4b67694ce86a2638474cdc2ded034a16c1b8626fbb86d0433daac1cf0fe +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f6624c75cbaec4c663b56fc6528fc340e41fc80 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fdfed2410dc7fe3b247b03ebc23899f1bbe68c4df084fd4ec73915e4644b135 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a609ba85328d58e09b8c4663f252b14f690499b --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3dd645bc94e56b1a80e95e9b85dc4b3787d169af3778f201e6bc2e58c3dfd4d +size 205568034 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..49f507c9df4f721022030b46083d0664d8f4b638 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61e1528240c9b0d20c231ea358cb4b4b139f156f57c01d6db0e885c11719b256 +size 205568290 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..034267741108b090779eb93d9b36f6a8025f86ee --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:addaf3c4432602bd42de6ca078ae3669652f7bb95a8c0c26ccb830e39d47a8ca +size 205568023 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96d527a9b0e384e6b7855513addbdfcd23aaca8e --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69bb32ad28dd2eea95a90dd6eaefb4855c0b444e9642c1c6dfbc0f6a8b8bacb5 +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3d574cf5030b180350c1511fcdee374b564ecf3 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17e6087018e2cd6f8e862c5e5856cceb4d99673ae384ca84a20b8a5af54ed345 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cf4c81b2946c0d924268c0dd28b2dce38fa8bf4 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b778e127b15e51f8db2c7c8381c03503d661b4a4ededce1dd54956092526c92 +size 205568290 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8eaad2e1a00b63183db8432fd3584c7a3346988f --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2f21d03dcded161b35a75f5056c2bb7c9d0d8512461a0ea8016159cf643d586 +size 205568034 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..879e3d946e100477f241f53f829f58db64e24bc1 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f4d363cfc806a949d80bc2bcfe8f5c14391ed568075b98f983dae85dc2d8cd +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..681168aef55b58f4386ef87c78dc731e35062ca1 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9937c9291e509968c1fa2091b4fb2f92f50197849cc53e1dffb372755d8b809a +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e9ab13f90259b744fbc3e4c05335fc118fc2bf5 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1742ebaa9900081aac42f9a8daaad96cca6171ec2e34f1a1299d98e2e180d04e +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77661fcdca0f579af817fd9c2a5e8173294881bc --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01e83f1d1f07d7f43e0cab2787a79e31041f72b09c98bee924dd205bee5e8f54 +size 205568290 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..665f15075baf6cc50ccf7ac5bd28a58827845a73 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b874a0a83ca905cc021c9a4f7025072427b35b6e500ccc01683a234df133e277 +size 205568098 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57581e852d67fec1d4200b5753e811f4baa300a4 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:879b36143783889679238f64de0cfee1a258272f576cf6989a517e678958dbd2 +size 205568226 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4df6c7d33f948ef9809c4b78a64e95745cb4cf07 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27cf6986c01aac9d6816c0dee17812d7faeab1da7b0d15403a8d7af693987668 +size 205568023 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a2371cf0516626e355b85630e786e40d119983d --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:846650e001c5ec878cf86128c3a437638edb23615a97ba952ad8d3b3e485d45f +size 205568162 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ef4da9eb445be03aabe41e0a21d28f79419975b --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1a72742377b2b0b826e144b7fb04cf16ac4a0c094e827863fb0ddc5daf752f8 +size 205568098 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e6f6bb5c6b70c509c9111b6ee586501c13e1276 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cd317dba4bbf4404cca010d6fc20c5d2088fc24edda79eb8963ee7303a891b3 +size 205568290 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56d5db4b9739f9160b9bc8ea43608fe0f1c339e5 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf35ee42524a59af205410a8fa6612830ece60de8322067a37003fd046abf415 +size 205568034 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afb44e9ce32e07b6c445c5b6b2757d29ada9a80e --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea3a441b4c32d1d787aed7575bacf18eae77ce136b6af669b12f6dcee79de7e7 +size 205568151 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..326539dfe95d4d495f3e3be65b8bc20575da6f59 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:138730fe4f2772f092f2cd8536b14fe101545bc92af16d5d2c665da7f554aa25 +size 205568151 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f13c59c599fa24e935249b087d7234f5d4810f6 --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:561abba7315646722fe176b793f66cd6b51c733472a9d70f09e046e124314945 +size 205568151 diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..766507589dbb99e2efccdfce89e43178ed80580e --- /dev/null +++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94bef5aeb9008989ad07e90be2e9dc49c1683ec2b8993504f88acb37375f173e +size 205568151 diff --git a/1b11b51b5/global_step2891/layer_01-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f22a37bf00cc86afb7ce7278cdfaf0178cc96756 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:935e7137339da3c98f016f1790e443605d9fd6e1d368cc811253260de8aa3522 +size 187630851 diff --git a/1b11b51b5/global_step2891/layer_03-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdbd687987c97e78a3defa555f78285544d77515 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02c74b5a94b9b38d61ac82bcae0a7436e6ae2683bd24fc2d3a616499f1a526aa +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_04-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5055834624bb9ca7b5489287442d17af14f07f63 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fe8b8fa7de105de10893eb33ec0a01529a2d82488d1944eba7e4d7368ede3df +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_05-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e92eae7e1a508cba1a76d13c9911d5c1bad631b3 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ace71618cf90db5137476f2d6ca7919aa2650158a5058bf6f1a05e0127602a7d +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_06-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6fb2eb2cd83367e5a5be429239c4036deb32a80 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6624dcb3e4f668e8d19871220af1376c4ade3d4c2ab9a283eb334be642be3fe +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_07-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1a46cb7c5f66731aad0ba5c5adaca521746bb6e --- /dev/null +++ b/1b11b51b5/global_step2891/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f8963f06713656769ef2087ebf6bf9ed70a0a2a22b746e2132e6a45af6d070 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_08-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..762161a5c209defb2af50788fd62aa3b37bb339c --- /dev/null +++ b/1b11b51b5/global_step2891/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d17e443969a89ed0893ecaa6f978fd74ed65321fc0404e468e5911757e3049b +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_09-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..274d997741c7f03c485b735a1e45931d123b1b5b --- /dev/null +++ b/1b11b51b5/global_step2891/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e9ab2a41923f078c8638da37482fdf175c1388a88a075bd1d46f23afac992ac +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_10-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f94da92c045a7b526d98e894f63e02355fe3aa00 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1594a755b472217ba3717692ac9a69175d35fa81e143237aef1ab769decb70f0 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_11-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d174e8e999f4b7f71ef3186e71b58ace1b699d4 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6176274d88d691670e6e52d2dbee85aeed776cee67371440ac7a84afdfc21f80 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_12-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4cf03f768db8d72a7ff4951ed6523856d4f1504 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:557b106d7a92bc5dfc0e72aa0a7095a1040d8f9d16860c1a55d02ccfd955a8cc +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_13-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e9d7f46bc765206e8be20c2704156062fd0d121 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfbcc5aa6fa103a8996658fdc5a0a138e7797491be9a326789baed827b1cecb2 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_14-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9b91f6678922194149be6d5f53d09668b0df506 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d36f9442e5fbb5e55ba18c9092693f7f857dce127305037f08b0a4775e68d711 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_15-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..608a1b3a36a89912d361428b95a4c1558f595e1a --- /dev/null +++ b/1b11b51b5/global_step2891/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd0af05b4dba722d354041a7b025716680db2ad4e16aca81bf572744b27f66b +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_16-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a390db7e2203a9a1b711afef7669c41d10b480c8 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:368978b303c7f1e2abeeeb9d45c3a83858455f303e475f0d54fd5b70186f4942 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_17-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5642ae8c8b1fb0c264f11a95c07b4954d78bcbf --- /dev/null +++ b/1b11b51b5/global_step2891/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7808953866c72a997eede64108f9e3a55d9286ca6ce850d019a30f2706fc982b +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_18-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..630db0dbaf28187c183f1bf536162802cf1818a5 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0799499a11114172b29437a4640555650d200ac9709cd72e490b99a5ebf7c75 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_19-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4723e90a9161c68498432efbf83da1398375b326 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441228f18cc2f6b3d8d171bfd51783ba6373e6e894ce8d4780270dd96958cff0 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_20-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a62c608f2ae1e1c658ffdc6fecb2a80b59e9be1c --- /dev/null +++ b/1b11b51b5/global_step2891/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8140a0c4b7f1bc22db24f90071f7ac915653e05eb0f7f4676a29477e144db73 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_21-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0db81c1f56651ee30ac2a5d6e705896a2a04eeb7 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60256e1e9854635f42a562bb600de33e1b3e92fb95a83e98a264c98bdab8871 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_22-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c0040c4d6010bda90bb59b6bbf22e103234f083 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c70fdb8d577a99393fd62ae8764e4cf8b4e070050c0343304a88254597a2f3 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_23-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f0d1d2ffa9265334e8a1912a8a813631a172b05 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e8de3f57bb8c7584d92634840143b3adba867ecb31213f8c294bfdd472951bf +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_24-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ba52133de6ce1d05ebb374a433c6f3b26a9043c --- /dev/null +++ b/1b11b51b5/global_step2891/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6048ea11d87e11eb6707efdb2e9938019cb98d01c0f42d9d719609922859a4d +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_25-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09358f58d0d69f96103c485eb7cf8c45c047c50a --- /dev/null +++ b/1b11b51b5/global_step2891/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f2e81c1622023c8f5f6a54c81091aeef23e8a4bf6a7babc3d874928466adcda +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_26-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70a84fbeb1814fb76f69464874c939129bb7c50b --- /dev/null +++ b/1b11b51b5/global_step2891/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:833fa4cfc0038eee86ee2ab9bfbd671929e1e236b8c5f9590cfcfca2b91e6d4d +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_27-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d3d89b81a674784d4c3f1f8002a3f4e4eee9c30 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:397343c660055deda4dccc91c22e1344151d42177d8c730efb230de323ff6be3 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_28-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9795f73f728b9ece2f79e8027573d53b6e2ca60c --- /dev/null +++ b/1b11b51b5/global_step2891/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0713b40860a46223dc0c0a18ff2b3197974ec3d40e24004be2f36471958f25b0 +size 77121283 diff --git a/1b11b51b5/global_step2891/layer_30-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..66c6d3031e694cf0262c3287332bf69165678bd9 --- /dev/null +++ b/1b11b51b5/global_step2891/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2eedd140b7a228f76e33a196f216985508d2a3ca725a8afaa2b038d8d4b0175 +size 8387 diff --git a/1b11b51b5/global_step2891/mp_rank_00_model_states.pt b/1b11b51b5/global_step2891/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c03362f7d58012b0bd75965c7895dc91aae6b2b0 --- /dev/null +++ b/1b11b51b5/global_step2891/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f361656d10f17d839baa18dde70fe3a595dd038e457098bf2ad1868ef8c95212 +size 43827 diff --git a/1b11b51b5/logs/2809896.err b/1b11b51b5/logs/2809896.err new file mode 100644 index 0000000000000000000000000000000000000000..8886be7c2f79b7ca7ee6ceefaebccbee741a8428 --- /dev/null +++ b/1b11b51b5/logs/2809896.err @@ -0,0 +1,1121 @@ +6: 2023-02-05 02:32:06.991466: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-05 02:32:06.991575: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.017666: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.056489: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-05 02:32:07.115054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.115061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.115081: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.115081: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.122419: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.126554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.126583: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-05 02:32:07.128135: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.128175: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.165478: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.165506: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-05 02:32:07.247320: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.247347: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.250819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.250844: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-05 02:32:07.259697: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.259724: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-05 02:32:07.281499: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-05 02:32:07.283011: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-05 02:32:07.283059: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.375989: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.403292: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.403320: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.403594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.403621: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.424432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.424459: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.430879: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.488958: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.495842: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-05 02:32:07.505564: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-05 02:32:07.507092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.507120: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.553425: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.553451: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-05 02:32:07.613314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.613341: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-05 02:32:07.616406: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:07.616433: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.624838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.624867: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-05 02:32:07.652685: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-05 02:32:07.781082: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:07.781108: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-05 02:32:08.010722: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.010837: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.010846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.010859: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-05 02:32:08.010958: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.010970: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-05 02:32:08.034719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.034753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.034859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.034871: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-05 02:32:08.034889: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.034900: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.035427: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.035563: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.035575: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.035584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.035716: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.035728: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-05 02:32:08.113008: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.113129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.113140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.140502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.140617: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.140628: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-05 02:32:08.202746: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.202858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.202868: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.307366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.307486: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.307496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.315002: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.315122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.315132: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-05 02:32:08.319880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.320003: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.320014: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.336640: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.336757: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.336768: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-05 02:32:08.349755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.349875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-05 02:32:08.349886: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.358988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.359105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.359116: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-05 02:32:08.475983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.476097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-05 02:32:08.476107: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-05 02:32:11.682289: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-05 02:32:11.682316: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682321: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682319: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682341: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: 2023-02-05 02:32:11.682331: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-05 02:32:11.682334: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-05 02:32:11.682339: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682388: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-05 02:32:11.682349: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682419: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682432: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-05 02:32:11.682359: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-05 02:32:11.682360: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682443: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:11.682445: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683371: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683386: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683410: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683426: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683450: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683489: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683496: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-05 02:32:11.683505: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683827: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683819: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683852: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683863: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683873: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683884: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683889: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-05 02:32:11.683895: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684365: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684366: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684397: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684425: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684444: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684461: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684471: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-05 02:32:11.684476: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.685976: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.685983: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.685974: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.686056: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.686064: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.686070: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.686073: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-05 02:32:11.686058: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-05 02:32:23.387173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.387208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.387230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.387244: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.387251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.387257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.387277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.387284: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:23.388071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:23.388090: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:23.388124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:23.388118: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:23.388142: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-05 02:32:23.388151: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 2023-02-05 02:32:23.388139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:23.388110: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:23.388162: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388128: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:23.388182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.388179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.400567: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-05 02:32:23.388126: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-05 02:32:23.400595: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-05 02:32:23.388264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.388238: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.388275: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.388218: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.400655: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-05 02:32:23.388173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-05 02:32:23.388315: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:23.400818: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.388157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.400659: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-05 02:32:23.388194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-05 02:32:23.400674: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-05 02:32:23.400691: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.388282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 2023-02-05 02:32:23.388210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:23.400713: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-05 02:32:23.400764: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:23.388219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:23.400856: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-05 02:32:23.388229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:23.400881: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-05 02:32:23.388237: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:23.388223: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 2023-02-05 02:32:23.388225: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:23.400896: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:23.400916: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.388229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 2023-02-05 02:32:23.400940: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-05 02:32:23.400955: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-05 02:32:23.400964: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:23.388242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:23.388290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:23.388287: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:23.388332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:23.388309: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:23.404123: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404367: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404396: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404399: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:23.404157: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:23.404178: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:23.404192: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:23.404201: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:23.404209: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404434: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:23.404227: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:23.404231: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404443: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404455: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404483: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404442: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404445: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-05 02:32:23.404682: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404481: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404503: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404521: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404541: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404548: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404478: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404549: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404498: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404518: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404545: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404551: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404557: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-05 02:32:23.404569: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-05 02:32:23.404793: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-05 02:32:48.209678: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.209703: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.209715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.209728: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.209739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.209755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: 2023-02-05 02:32:48.210053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-05 02:32:48.210054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.209756: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.209898: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.209821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.209922: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.210102: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.209939: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.210089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-05 02:32:48.210095: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.209975: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.210117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.209972: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.210136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.209983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.210118: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-05 02:32:48.210153: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.209988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.210132: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-05 02:32:48.210171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-05 02:32:48.210000: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.210142: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-05 02:32:48.210175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.210357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-05 02:32:48.210260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.210376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216522: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216521: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216522: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216524: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.216712: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-05 02:32:48.216525: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216544: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-05 02:32:48.216545: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216714: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-05 02:32:48.216544: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-05 02:32:48.216544: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-05 02:32:48.216546: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-05 02:32:48.216546: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-05 02:32:48.216549: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-05 02:32:48.216549: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.216715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.216718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.216728: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216721: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.216729: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216730: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216724: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.216738: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216739: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216742: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216745: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-05 02:32:48.216764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-05 02:32:48.216779: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.217981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.217983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.217984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.217985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.217988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.217990: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.217992: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.217996: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-05 02:32:48.218011: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.218011: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.218012: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.218015: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.218016: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.218017: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.218019: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-05 02:32:48.218022: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-05 02:32:48.235568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.235607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235546: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-05 02:32:48.235617: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.235645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.235657: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.235670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.235675: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.235681: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235577: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.235688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237975: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-05 02:32:48.237970: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-05 02:32:48.238048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237973: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-05 02:32:48.238050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-05 02:32:48.238055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237976: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-05 02:32:48.238056: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237987: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-05 02:32:48.237988: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-05 02:32:48.237992: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-05 02:32:48.237992: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-05 02:32:48.237995: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: 2023-02-05 02:32:48.237997: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-05 02:32:48.238000: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.238060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.238063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.238077: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238066: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-05 02:32:48.238079: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238083: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238086: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238086: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238086: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238090: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-05 02:32:48.238092: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217395: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217397: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217397: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217397: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217398: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217402: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217407: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-05 02:32:48.217416: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217416: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217417: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217419: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217420: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217421: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217423: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-05 02:32:48.217424: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_upper_triang_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +5: Successfully preprocessed all matching files. +5: Successfully preprocessed all matching files. +5: Successfully preprocessed all matching files. +4: Successfully preprocessed all matching files. +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +4: Building extension module utils... +4: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: +1: +1: +1: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: +3: +3: +3: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: +6: +6: +6: +6: +6: +4: Loading extension module utils... +0: Loading extension module utils... +2: Loading extension module utils... +7: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +0: +0: Loading extension module utils...Loading extension module utils... +0: +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +6: +6: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: +4: +4: Loading extension module utils...Loading extension module utils...Loading extension module utils...Loading extension module utils... +4: +4: +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +3: +3: Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +3: +3: +4: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +4: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils...Loading extension module utils... +2: +2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: +7: Loading extension module utils...Loading extension module utils...Loading extension module utils... +7: +7: +7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: Loading extension module utils...Loading extension module utils... +7: +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +7: +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +5: +5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +5: +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +1: +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/1b11b51b5/logs/2809896.out b/1b11b51b5/logs/2809896.out new file mode 100644 index 0000000000000000000000000000000000000000..3f88b79ce5eaabc1fa4fd871b9613969426144d5 --- /dev/null +++ b/1b11b51b5/logs/2809896.out @@ -0,0 +1,8491 @@ +Model parameters: d_model 1792 ffw_size 7168 kv_size 128 n_heads 14 n_layers 26 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 26 --hidden-size 1792 --num-attention-heads 14 --kv-channels 128 --ffn-hidden-size 7168 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --loss-scale 12 --clip-grad 1.0 --kill-switch-path kill-switch-1b11b51b5val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --no-load-optim --reset-progress --override-lr-scheduler --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-only true --eval-iters 100 --tensorboard-dir tensorboard_1b11b51b5val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_1b11b51b5 --load checkpoints_1b11b51b5 --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/2809896.json --zero-stage 0 +START 2809896: Sun Feb 5 02:30:43 EET 2023 +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 42.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 41.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 45.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 40.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 38.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 49.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 42.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 40.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 45.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 41.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 35.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 40.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 34.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 38.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +7: Launching on nid005024 (7/8), master nid005017 port 9999, GPUs 8, CUDA: True +6: Launching on nid005023 (6/8), master nid005017 port 9999, GPUs 8, CUDA: True +1: Launching on nid005018 (1/8), master nid005017 port 9999, GPUs 8, CUDA: True +4: Launching on nid005021 (4/8), master nid005017 port 9999, GPUs 8, CUDA: True +5: Launching on nid005022 (5/8), master nid005017 port 9999, GPUs 8, CUDA: True +3: Launching on nid005020 (3/8), master nid005017 port 9999, GPUs 8, CUDA: True +0: Launching on nid005017 (0/8), master nid005017 port 9999, GPUs 8, CUDA: True +2: Launching on nid005019 (2/8), master nid005017 port 9999, GPUs 8, CUDA: True +7: > setting tensorboard ... +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/2809896.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... True +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 7168 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1792 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-1b11b51b5val +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_1b11b51b5 +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... 12.0 +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 14 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 26 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_1b11b51b5 +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_1b11b51b5val +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-05 02:33:57,034] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.098 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 87 +0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.o scaled_upper_triang_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 63 +0: ninja: no work to do. +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: ninja: no work to do. +0: >>> done with compiling and loading fused kernels. Compilation time: 26.553 seconds +0: time to initialize megatron (seconds): 146.360 +0: [after megatron is initialized] datetime: 2023-02-05 02:34:26 +0: building GPT model ... +0: [2023-02-05 02:34:26,508] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-05 02:34:26,509] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-05 02:34:26,509] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.47 GB, percent = 6.3% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-05 02:34:28,489] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=33 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: ParallelTransformerLayerPipe +0: 23: ParallelTransformerLayerPipe +0: 24: ParallelTransformerLayerPipe +0: 25: ParallelTransformerLayerPipe +0: 26: ParallelTransformerLayerPipe +0: 27: ParallelTransformerLayerPipe +0: 28: ParallelTransformerLayerPipe +0: 29: undo +0: 30: MixedFusedLayerNorm +0: 31: EmbeddingPipe +0: 32: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-05 02:34:28,858] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-05 02:34:28,859] [INFO] [utils.py:828:see_memory_usage] MA 2.05 GB Max_MA 2.05 GB CA 2.19 GB Max_CA 2 GB +0: [2023-02-05 02:34:28,859] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.52 GB, percent = 6.3% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-05 02:34:28,861] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-05 02:34:42,192] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-05 02:34:42,193] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-05 02:34:42,193] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-05 02:34:42,204] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-05 02:34:42,204] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-05 02:34:42,311] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-05 02:34:42,312] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB Max_MA 2.06 GB CA 2.19 GB Max_CA 2 GB +0: [2023-02-05 02:34:42,312] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.21 GB, percent = 6.4% +4: ninja: no work to do. +0: Time to load utils op: 0.15061402320861816 seconds +0: Time to load utils op: 0.15090179443359375 seconds +0: Time to load utils op: 0.19407129287719727 seconds +0: Time to load utils op: 0.15023088455200195 secondsTime to load utils op: 0.15064501762390137 secondsTime to load utils op: 0.1507422924041748 seconds +0: +0: +0: Time to load utils op: 0.15055227279663086 seconds +0: Time to load utils op: 0.15049982070922852 seconds +2: Time to load utils op: 0.27375149726867676 seconds +2: Time to load utils op: 0.14833402633666992 seconds +2: Time to load utils op: 0.1490778923034668 seconds +2: Time to load utils op: 0.14825797080993652 secondsTime to load utils op: 0.14822840690612793 seconds +2: +2: Time to load utils op: 0.1482524871826172 seconds +2: Time to load utils op: 0.14826107025146484 seconds +2: Time to load utils op: 0.14829063415527344 seconds +4: Time to load utils op: 0.14612770080566406 secondsTime to load utils op: 0.14724159240722656 seconds +4: +4: Time to load utils op: 0.145674467086792 seconds +4: Time to load utils op: 0.14727401733398438 seconds +4: Time to load utils op: 0.1460864543914795 seconds +4: Time to load utils op: 0.14700007438659668 seconds +4: Time to load utils op: 0.1470193862915039 seconds +4: Time to load utils op: 0.28795838356018066 seconds +3: Time to load utils op: 0.1580641269683838 seconds +3: Time to load utils op: 0.1580677032470703 seconds +3: Time to load utils op: 0.1582660675048828 seconds +3: Time to load utils op: 0.15827178955078125 seconds +3: Time to load utils op: 0.15828704833984375 seconds +3: Time to load utils op: 0.15880703926086426 seconds +3: Time to load utils op: 0.15883946418762207 secondsTime to load utils op: 0.15882349014282227 seconds +3: +1: Time to load utils op: 0.16072487831115723 seconds +1: Time to load utils op: 0.16073369979858398 seconds +1: Time to load utils op: 0.1607356071472168 seconds +1: Time to load utils op: 0.1607496738433838 seconds +1: Time to load utils op: 0.16074895858764648 secondsTime to load utils op: 0.16075730323791504 secondsTime to load utils op: 0.1607685089111328 secondsTime to load utils op: 0.1607673168182373 seconds +1: +1: +1: +7: Time to load utils op: 0.15805459022521973 secondsTime to load utils op: 0.15678167343139648 seconds +7: +7: Time to load utils op: 0.1563711166381836 seconds +7: Time to load utils op: 0.15667986869812012 secondsTime to load utils op: 0.15596556663513184 seconds +7: +7: Time to load utils op: 0.1567378044128418 seconds +7: Time to load utils op: 0.28981828689575195 seconds +7: Time to load utils op: 0.15645766258239746 seconds +6: Time to load utils op: 0.1561574935913086 seconds +6: Time to load utils op: 0.15617632865905762 seconds +6: Time to load utils op: 0.1562037467956543 seconds +6: Time to load utils op: 0.15662050247192383 secondsTime to load utils op: 0.15662264823913574 seconds +6: +6: Time to load utils op: 0.1566324234008789 seconds +6: Time to load utils op: 0.1567087173461914 seconds +5: Time to load utils op: 0.16223931312561035 secondsTime to load utils op: 0.16388249397277832 seconds +5: +5: Time to load utils op: 0.16389155387878418 seconds +6: Time to load utils op: 0.15671277046203613 seconds +5: Time to load utils op: 0.16350030899047852 seconds +5: Time to load utils op: 0.1631937026977539 seconds +5: Time to load utils op: 0.16251301765441895 seconds +5: Time to load utils op: 0.16286206245422363 seconds +5: Time to load utils op: 0.1630856990814209 seconds +0: [2023-02-05 02:34:42,602] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-02-05 02:34:42,602] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB Max_MA 2.04 GB CA 2.19 GB Max_CA 2 GB +0: [2023-02-05 02:34:42,603] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.22 GB, percent = 6.4% +0: Time to load utils op: 0.0005381107330322266 seconds +0: Time to load utils op: 0.0005221366882324219 seconds +0: Time to load utils op: 0.00055694580078125 seconds +0: Time to load utils op: 0.0006151199340820312 secondsTime to load utils op: 0.0006003379821777344 secondsTime to load utils op: 0.0006165504455566406 seconds +0: +0: +0: Time to load utils op: 0.0006096363067626953 seconds +6: Time to load utils op: 0.0007691383361816406 seconds +6: Time to load utils op: 0.0011408329010009766 seconds +3: Time to load utils op: 0.0012526512145996094 seconds +6: Time to load utils op: 0.0013625621795654297 seconds +6: Time to load utils op: 0.0012745857238769531 seconds +4: Time to load utils op: 0.00047779083251953125 seconds +6: Time to load utils op: 0.0012981891632080078 secondsTime to load utils op: 0.0012695789337158203 seconds +6: +4: Time to load utils op: 0.0005197525024414062 seconds +6: Time to load utils op: 0.0012717247009277344 seconds +6: Time to load utils op: 0.0012645721435546875 seconds +4: Time to load utils op: 0.0004825592041015625 seconds +4: Time to load utils op: 0.0004584789276123047 secondsTime to load utils op: 0.0004439353942871094 secondsTime to load utils op: 0.00046539306640625 secondsTime to load utils op: 0.0004458427429199219 seconds +4: +4: +4: +4: Time to load utils op: 0.0004813671112060547 seconds +3: Time to load utils op: 0.001659393310546875 seconds +3: Time to load utils op: 0.00147247314453125 secondsTime to load utils op: 0.0016329288482666016 seconds +3: +3: Time to load utils op: 0.0016407966613769531 seconds +3: Time to load utils op: 0.0016818046569824219 seconds +3: Time to load utils op: 0.0016853809356689453 seconds +3: Time to load utils op: 0.001623392105102539 seconds +2: Time to load utils op: 0.0005652904510498047 seconds +2: Time to load utils op: 0.0005426406860351562 secondsTime to load utils op: 0.0005252361297607422 seconds +2: +2: Time to load utils op: 0.0005772113800048828 seconds +2: Time to load utils op: 0.000637054443359375 seconds +2: Time to load utils op: 0.0006256103515625 seconds +2: Time to load utils op: 0.0006151199340820312 seconds +2: Time to load utils op: 0.0006196498870849609 seconds +7: Time to load utils op: 0.0005898475646972656 seconds +7: Time to load utils op: 0.0005943775177001953 seconds +7: Time to load utils op: 0.0006337165832519531 seconds +7: Time to load utils op: 0.0006198883056640625 seconds +7: Time to load utils op: 0.000640869140625 seconds +7: Time to load utils op: 0.0006756782531738281 secondsTime to load utils op: 0.0006389617919921875 seconds +7: +7: Time to load utils op: 0.0006933212280273438 seconds +5: Time to load utils op: 0.0007336139678955078 seconds +5: Time to load utils op: 0.0012247562408447266 seconds +5: Time to load utils op: 0.0012772083282470703 seconds +5: Time to load utils op: 0.001188039779663086 seconds +5: Time to load utils op: 0.0012052059173583984 seconds +5: Time to load utils op: 0.0012354850769042969 seconds +5: Time to load utils op: 0.0012421607971191406 seconds +5: Time to load utils op: 0.0012013912200927734 seconds +1: Time to load utils op: 0.0006010532379150391 seconds +1: Time to load utils op: 0.0006742477416992188 secondsTime to load utils op: 0.000774383544921875 seconds +1: +1: Time to load utils op: 0.0007770061492919922 seconds +1: Time to load utils op: 0.0013883113861083984 seconds +1: Time to load utils op: 0.0013172626495361328 secondsTime to load utils op: 0.0014536380767822266 seconds +1: +1: Time to load utils op: 0.0014028549194335938 seconds +0: [2023-02-05 02:34:42,734] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-05 02:34:42,734] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB Max_MA 4.24 GB CA 5.44 GB Max_CA 5 GB +0: [2023-02-05 02:34:42,734] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:42,829] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-05 02:34:42,830] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB Max_MA 4.24 GB CA 5.44 GB Max_CA 5 GB +0: [2023-02-05 02:34:42,830] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:42,927] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-05 02:34:42,927] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-02-05 02:34:42,928] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:43,022] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-05 02:34:43,022] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-02-05 02:34:43,023] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:43,121] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-05 02:34:43,121] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-02-05 02:34:43,121] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:43,215] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-05 02:34:43,216] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB Max_MA 6.19 GB CA 8.31 GB Max_CA 8 GB +0: [2023-02-05 02:34:43,216] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:43,316] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-05 02:34:43,317] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB Max_MA 6.32 GB CA 8.34 GB Max_CA 8 GB +0: [2023-02-05 02:34:43,317] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:43,412] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-05 02:34:43,412] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB Max_MA 6.32 GB CA 8.34 GB Max_CA 8 GB +0: [2023-02-05 02:34:43,413] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 32.37 GB, percent = 6.4% +0: [2023-02-05 02:34:43,413] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-05 02:34:43,413] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-05 02:34:43,413] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-05 02:34:43,413] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-05 02:34:43,413] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-05 02:34:43,414] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-05 02:34:43,415] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-05 02:34:43,416] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-05 02:34:43,416] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-05 02:34:43,416] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.0004210472106933594 seconds +0: [2023-02-05 02:34:43,416] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-05 02:34:43,466] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=33 [0, 33) STAGE_PARAMS=1096338432 (1096.338M) TOTAL_PARAMS=1096338432 (1096.338M) UNIQUE_PARAMS=1096338432 (1096.338M) +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +3: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +6: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +4: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt... +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +5: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +4: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +3: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +7: [2023-02-05 02:34:43,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +6: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt. +1: [2023-02-05 02:34:43,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +6: [2023-02-05 02:34:43,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +4: [2023-02-05 02:34:43,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +7: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +2: [2023-02-05 02:34:43,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +3: [2023-02-05 02:34:43,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +1: [2023-02-05 02:34:43,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +0: [2023-02-05 02:34:43,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt... +5: [2023-02-05 02:34:43,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:43,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:43,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +4: [2023-02-05 02:34:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +2: [2023-02-05 02:34:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:43,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:43,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:43,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:43,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:43,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:43,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +5: [2023-02-05 02:34:43,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:43,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:43,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:43,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:43,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +1: [2023-02-05 02:34:43,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +7: [2023-02-05 02:34:43,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +6: [2023-02-05 02:34:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:43,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:43,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:43,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +0: [2023-02-05 02:34:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt. +3: [2023-02-05 02:34:43,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:43,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:43,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:43,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:44,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:44,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +0: [2023-02-05 02:34:44,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:44,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +3: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +2: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +4: [2023-02-05 02:34:44,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:44,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:44,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:44,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +7: [2023-02-05 02:34:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +6: [2023-02-05 02:34:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +1: [2023-02-05 02:34:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt... +5: [2023-02-05 02:34:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +0: [2023-02-05 02:34:44,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +2: [2023-02-05 02:34:44,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +5: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +7: [2023-02-05 02:34:44,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +1: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +6: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +3: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt. +4: [2023-02-05 02:34:44,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +5: [2023-02-05 02:34:44,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +5: [2023-02-05 02:34:44,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +4: [2023-02-05 02:34:44,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +0: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +7: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +2: [2023-02-05 02:34:44,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +6: [2023-02-05 02:34:44,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +1: [2023-02-05 02:34:44,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt... +3: [2023-02-05 02:34:44,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +3: [2023-02-05 02:34:44,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +7: [2023-02-05 02:34:44,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +4: [2023-02-05 02:34:44,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +1: [2023-02-05 02:34:44,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +6: [2023-02-05 02:34:44,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +2: [2023-02-05 02:34:44,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt. +0: [2023-02-05 02:34:44,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:44,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:44,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +7: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +6: [2023-02-05 02:34:44,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +3: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +2: [2023-02-05 02:34:44,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +0: [2023-02-05 02:34:44,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +0: [2023-02-05 02:34:44,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:44,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:44,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:44,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +1: [2023-02-05 02:34:44,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +4: [2023-02-05 02:34:44,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt... +5: [2023-02-05 02:34:44,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:44,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:44,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:44,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +7: [2023-02-05 02:34:44,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:44,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:44,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:44,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:44,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:44,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:44,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +1: [2023-02-05 02:34:44,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:44,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:44,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +2: [2023-02-05 02:34:44,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:44,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +5: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +4: [2023-02-05 02:34:44,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:44,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:44,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:44,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +3: [2023-02-05 02:34:44,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:44,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:44,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt. +6: [2023-02-05 02:34:44,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:44,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:44,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:44,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:44,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:44,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:44,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:44,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:45,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:45,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:45,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +7: [2023-02-05 02:34:45,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +6: [2023-02-05 02:34:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +0: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +5: [2023-02-05 02:34:45,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +4: [2023-02-05 02:34:45,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +3: [2023-02-05 02:34:45,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:45,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:45,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:45,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:45,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +1: [2023-02-05 02:34:45,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt... +2: [2023-02-05 02:34:45,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +6: [2023-02-05 02:34:45,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +2: [2023-02-05 02:34:45,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +0: [2023-02-05 02:34:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +5: [2023-02-05 02:34:45,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +7: [2023-02-05 02:34:45,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +3: [2023-02-05 02:34:45,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +4: [2023-02-05 02:34:45,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt. +1: [2023-02-05 02:34:45,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:45,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +6: [2023-02-05 02:34:45,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +4: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +1: [2023-02-05 02:34:45,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +3: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +0: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +5: [2023-02-05 02:34:45,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +2: [2023-02-05 02:34:45,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +0: [2023-02-05 02:34:45,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:45,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:45,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:45,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:45,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:45,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +4: [2023-02-05 02:34:45,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +2: [2023-02-05 02:34:45,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:45,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:45,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:45,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +3: [2023-02-05 02:34:45,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:45,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +1: [2023-02-05 02:34:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:45,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:45,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +6: [2023-02-05 02:34:45,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:45,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:45,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:45,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:45,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:45,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:45,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:45,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:45,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +5: [2023-02-05 02:34:45,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:45,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt... +7: [2023-02-05 02:34:45,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:45,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt. +7: [2023-02-05 02:34:45,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:46,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:46,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:46,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:46,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:46,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:46,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +5: [2023-02-05 02:34:46,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:46,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:46,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:46,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:46,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +0: [2023-02-05 02:34:46,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:46,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:46,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:46,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +6: [2023-02-05 02:34:46,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +3: [2023-02-05 02:34:46,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +7: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +2: [2023-02-05 02:34:46,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +1: [2023-02-05 02:34:46,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt... +4: [2023-02-05 02:34:46,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +4: [2023-02-05 02:34:46,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +2: [2023-02-05 02:34:46,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +3: [2023-02-05 02:34:46,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +0: [2023-02-05 02:34:46,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +5: [2023-02-05 02:34:46,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +1: [2023-02-05 02:34:46,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +6: [2023-02-05 02:34:46,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt. +7: [2023-02-05 02:34:46,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +5: [2023-02-05 02:34:46,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +3: [2023-02-05 02:34:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +6: [2023-02-05 02:34:46,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +5: [2023-02-05 02:34:46,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +0: [2023-02-05 02:34:46,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +2: [2023-02-05 02:34:46,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +7: [2023-02-05 02:34:46,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +3: [2023-02-05 02:34:46,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +0: [2023-02-05 02:34:46,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +2: [2023-02-05 02:34:46,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +1: [2023-02-05 02:34:46,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt... +4: [2023-02-05 02:34:46,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +6: [2023-02-05 02:34:46,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +4: [2023-02-05 02:34:46,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +7: [2023-02-05 02:34:46,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt. +1: [2023-02-05 02:34:46,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +7: [2023-02-05 02:34:46,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +0: [2023-02-05 02:34:46,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +6: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +2: [2023-02-05 02:34:46,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +4: [2023-02-05 02:34:46,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +3: [2023-02-05 02:34:46,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +4: [2023-02-05 02:34:46,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +6: [2023-02-05 02:34:46,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +2: [2023-02-05 02:34:46,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:46,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +7: [2023-02-05 02:34:46,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +3: [2023-02-05 02:34:46,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +1: [2023-02-05 02:34:46,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt... +5: [2023-02-05 02:34:46,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +1: [2023-02-05 02:34:46,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +5: [2023-02-05 02:34:46,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:46,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:46,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt. +0: [2023-02-05 02:34:46,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:46,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +0: [2023-02-05 02:34:46,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:46,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:46,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:46,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:46,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:46,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:46,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:46,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:46,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:46,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:46,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:46,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +0: [2023-02-05 02:34:46,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:46,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:46,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:46,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:46,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:46,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:46,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:46,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:46,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:46,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:46,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:46,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +4: [2023-02-05 02:34:46,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:46,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:46,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:46,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +4: [2023-02-05 02:34:46,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:46,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:46,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:46,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:46,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:46,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:46,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:46,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:46,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:46,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:46,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:46,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:46,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:47,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:47,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:47,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:47,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +7: [2023-02-05 02:34:47,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +6: [2023-02-05 02:34:47,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:47,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:47,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:47,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:47,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:47,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:47,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +5: [2023-02-05 02:34:47,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:47,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +5: [2023-02-05 02:34:47,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +6: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:47,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:47,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:47,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:47,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +2: [2023-02-05 02:34:47,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:47,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +1: [2023-02-05 02:34:47,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt... +3: [2023-02-05 02:34:47,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:47,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:47,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:47,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +7: [2023-02-05 02:34:47,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +3: [2023-02-05 02:34:47,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +2: [2023-02-05 02:34:47,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt. +1: [2023-02-05 02:34:47,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +0: [2023-02-05 02:34:47,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +3: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +2: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +7: [2023-02-05 02:34:47,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +1: [2023-02-05 02:34:47,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +6: [2023-02-05 02:34:47,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +0: [2023-02-05 02:34:47,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +4: [2023-02-05 02:34:47,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +5: [2023-02-05 02:34:47,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt... +5: [2023-02-05 02:34:47,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +6: [2023-02-05 02:34:47,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +2: [2023-02-05 02:34:47,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +4: [2023-02-05 02:34:47,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +7: [2023-02-05 02:34:47,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +3: [2023-02-05 02:34:47,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt. +1: [2023-02-05 02:34:47,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-02-05 02:34:47,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +6: [2023-02-05 02:34:47,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +0: [2023-02-05 02:34:47,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +0: [2023-02-05 02:34:47,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +7: [2023-02-05 02:34:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +4: [2023-02-05 02:34:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +5: [2023-02-05 02:34:47,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +7: [2023-02-05 02:34:47,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +4: [2023-02-05 02:34:47,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +3: [2023-02-05 02:34:47,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +2: [2023-02-05 02:34:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt... +1: [2023-02-05 02:34:47,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +1: [2023-02-05 02:34:47,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +6: [2023-02-05 02:34:47,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +5: [2023-02-05 02:34:47,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +3: [2023-02-05 02:34:47,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt. +2: [2023-02-05 02:34:47,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:47,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:47,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:47,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +4: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +1: [2023-02-05 02:34:47,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:47,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:47,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:47,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:47,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:47,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:47,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:47,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +5: [2023-02-05 02:34:47,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:47,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-02-05 02:34:47,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +6: [2023-02-05 02:34:47,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +7: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-02-05 02:34:47,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +0: [2023-02-05 02:34:47,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +2: [2023-02-05 02:34:47,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:47,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +2: [2023-02-05 02:34:47,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:47,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:47,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:47,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +6: [2023-02-05 02:34:47,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:47,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:47,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:47,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:47,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:47,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +7: [2023-02-05 02:34:47,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:47,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +1: [2023-02-05 02:34:47,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +4: [2023-02-05 02:34:47,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:47,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +5: [2023-02-05 02:34:47,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:47,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:47,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +0: [2023-02-05 02:34:48,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt... +3: [2023-02-05 02:34:48,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt. +3: [2023-02-05 02:34:48,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +0: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +0: [2023-02-05 02:34:48,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:48,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +6: [2023-02-05 02:34:48,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +5: [2023-02-05 02:34:48,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +4: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:48,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +7: [2023-02-05 02:34:48,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +2: [2023-02-05 02:34:48,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +3: [2023-02-05 02:34:48,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt... +1: [2023-02-05 02:34:48,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +7: [2023-02-05 02:34:48,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +2: [2023-02-05 02:34:48,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +3: [2023-02-05 02:34:48,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +1: [2023-02-05 02:34:48,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +4: [2023-02-05 02:34:48,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +6: [2023-02-05 02:34:48,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt. +5: [2023-02-05 02:34:48,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +0: [2023-02-05 02:34:48,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +0: [2023-02-05 02:34:48,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:48,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +6: [2023-02-05 02:34:48,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +3: [2023-02-05 02:34:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +7: [2023-02-05 02:34:48,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:48,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:48,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:48,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:48,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:48,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +2: [2023-02-05 02:34:48,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +3: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +1: [2023-02-05 02:34:48,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +7: [2023-02-05 02:34:48,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +5: [2023-02-05 02:34:48,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +2: [2023-02-05 02:34:48,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:48,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt... +4: [2023-02-05 02:34:48,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +4: [2023-02-05 02:34:48,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +6: [2023-02-05 02:34:48,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +5: [2023-02-05 02:34:48,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt. +1: [2023-02-05 02:34:48,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +5: [2023-02-05 02:34:48,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +3: [2023-02-05 02:34:48,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:48,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:48,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:48,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:48,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:48,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:48,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:48,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +3: [2023-02-05 02:34:48,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:48,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:48,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:48,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:48,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:48,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +0: [2023-02-05 02:34:48,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:48,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:48,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:48,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:48,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:48,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:48,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:48,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:48,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:48,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:48,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:48,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +0: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:48,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:48,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +1: [2023-02-05 02:34:48,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:48,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:48,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +6: [2023-02-05 02:34:48,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:48,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:48,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:48,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:48,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:48,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +6: [2023-02-05 02:34:49,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:49,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +1: [2023-02-05 02:34:49,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:49,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +5: [2023-02-05 02:34:49,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:49,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:49,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:49,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:49,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:49,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:49,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:49,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:49,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:49,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:49,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:49,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:49,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:49,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +4: [2023-02-05 02:34:49,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:49,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +7: [2023-02-05 02:34:49,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:49,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:49,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +2: [2023-02-05 02:34:49,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt... +4: [2023-02-05 02:34:49,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +2: [2023-02-05 02:34:49,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt. +7: [2023-02-05 02:34:49,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +0: [2023-02-05 02:34:49,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +0: [2023-02-05 02:34:49,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +1: [2023-02-05 02:34:49,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:49,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:49,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +3: [2023-02-05 02:34:49,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +4: [2023-02-05 02:34:49,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +3: [2023-02-05 02:34:49,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +1: [2023-02-05 02:34:49,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +4: [2023-02-05 02:34:49,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +7: [2023-02-05 02:34:49,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +5: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +2: [2023-02-05 02:34:49,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt... +6: [2023-02-05 02:34:49,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +5: [2023-02-05 02:34:49,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +6: [2023-02-05 02:34:49,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +2: [2023-02-05 02:34:49,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt. +7: [2023-02-05 02:34:49,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +3: [2023-02-05 02:34:49,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +0: [2023-02-05 02:34:49,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +6: [2023-02-05 02:34:49,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +7: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +5: [2023-02-05 02:34:49,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +4: [2023-02-05 02:34:49,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +1: [2023-02-05 02:34:49,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +0: [2023-02-05 02:34:49,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +1: [2023-02-05 02:34:49,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +3: [2023-02-05 02:34:49,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +4: [2023-02-05 02:34:49,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +5: [2023-02-05 02:34:49,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +6: [2023-02-05 02:34:49,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +7: [2023-02-05 02:34:49,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt... +2: [2023-02-05 02:34:49,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt. +2: [2023-02-05 02:34:49,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:49,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:49,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:49,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:49,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:49,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:49,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:49,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:49,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:49,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:49,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:49,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:49,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +7: [2023-02-05 02:34:49,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +0: [2023-02-05 02:34:49,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +0: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +6: [2023-02-05 02:34:49,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +7: [2023-02-05 02:34:49,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +1: [2023-02-05 02:34:49,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:49,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:49,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:49,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +1: [2023-02-05 02:34:49,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:49,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:49,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:49,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:49,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:49,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:49,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:49,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:49,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:49,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +6: [2023-02-05 02:34:49,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:49,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +4: [2023-02-05 02:34:49,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +2: [2023-02-05 02:34:49,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +3: [2023-02-05 02:34:49,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt... +5: [2023-02-05 02:34:49,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +3: [2023-02-05 02:34:49,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +4: [2023-02-05 02:34:49,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:49,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:49,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:49,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:49,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:49,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +5: [2023-02-05 02:34:49,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt. +2: [2023-02-05 02:34:49,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:49,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +3: [2023-02-05 02:34:50,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +3: [2023-02-05 02:34:50,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +4: [2023-02-05 02:34:50,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:50,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +4: [2023-02-05 02:34:50,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:50,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +0: [2023-02-05 02:34:50,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:50,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +0: [2023-02-05 02:34:50,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +2: [2023-02-05 02:34:50,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +7: [2023-02-05 02:34:50,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +5: [2023-02-05 02:34:50,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +6: [2023-02-05 02:34:50,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt... +1: [2023-02-05 02:34:50,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +2: [2023-02-05 02:34:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +5: [2023-02-05 02:34:50,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +1: [2023-02-05 02:34:50,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +6: [2023-02-05 02:34:50,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt. +7: [2023-02-05 02:34:50,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +0: [2023-02-05 02:34:50,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +0: [2023-02-05 02:34:50,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +3: [2023-02-05 02:34:50,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +2: [2023-02-05 02:34:50,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +7: [2023-02-05 02:34:50,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +3: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +4: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +7: [2023-02-05 02:34:50,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +2: [2023-02-05 02:34:50,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +5: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +4: [2023-02-05 02:34:50,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +1: [2023-02-05 02:34:50,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt... +6: [2023-02-05 02:34:50,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +1: [2023-02-05 02:34:50,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +5: [2023-02-05 02:34:50,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt. +6: [2023-02-05 02:34:50,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:50,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +7: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:50,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +0: [2023-02-05 02:34:50,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:50,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:50,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +5: [2023-02-05 02:34:50,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +3: [2023-02-05 02:34:50,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +4: [2023-02-05 02:34:50,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +1: [2023-02-05 02:34:50,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +3: [2023-02-05 02:34:50,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:50,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +0: [2023-02-05 02:34:50,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:50,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:50,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:50,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +7: [2023-02-05 02:34:50,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +2: [2023-02-05 02:34:50,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:50,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +4: [2023-02-05 02:34:50,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:50,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:50,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +1: [2023-02-05 02:34:50,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +5: [2023-02-05 02:34:50,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:50,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt... +6: [2023-02-05 02:34:50,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt. +6: [2023-02-05 02:34:50,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:50,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:51,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:51,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +3: [2023-02-05 02:34:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:51,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +0: [2023-02-05 02:34:51,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +3: [2023-02-05 02:34:51,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +7: [2023-02-05 02:34:51,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +4: [2023-02-05 02:34:51,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:51,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:51,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +7: [2023-02-05 02:34:51,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:51,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +1: [2023-02-05 02:34:51,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:51,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +2: [2023-02-05 02:34:51,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +5: [2023-02-05 02:34:51,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +2: [2023-02-05 02:34:51,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:51,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:51,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:51,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:51,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:51,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:51,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +6: [2023-02-05 02:34:51,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt... +4: [2023-02-05 02:34:51,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +1: [2023-02-05 02:34:51,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +6: [2023-02-05 02:34:51,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +0: [2023-02-05 02:34:51,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +5: [2023-02-05 02:34:51,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt. +0: [2023-02-05 02:34:51,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +4: [2023-02-05 02:34:51,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-02-05 02:34:51,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +0: [2023-02-05 02:34:51,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +4: [2023-02-05 02:34:51,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +5: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +7: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +3: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +1: [2023-02-05 02:34:51,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +2: [2023-02-05 02:34:51,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +7: [2023-02-05 02:34:51,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-02-05 02:34:51,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +1: [2023-02-05 02:34:51,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-02-05 02:34:51,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +2: [2023-02-05 02:34:51,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +3: [2023-02-05 02:34:51,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +5: [2023-02-05 02:34:51,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt... +6: [2023-02-05 02:34:51,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt. +6: [2023-02-05 02:34:51,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +3: [2023-02-05 02:34:51,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +0: [2023-02-05 02:34:51,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +3: [2023-02-05 02:34:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +0: [2023-02-05 02:34:51,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-02-05 02:34:51,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +2: [2023-02-05 02:34:51,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-02-05 02:34:51,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-02-05 02:34:51,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +4: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +2: [2023-02-05 02:34:51,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +4: [2023-02-05 02:34:51,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +7: [2023-02-05 02:34:51,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +6: [2023-02-05 02:34:51,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +5: [2023-02-05 02:34:51,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt... +1: [2023-02-05 02:34:51,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +5: [2023-02-05 02:34:51,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +7: [2023-02-05 02:34:51,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +1: [2023-02-05 02:34:51,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt. +6: [2023-02-05 02:34:51,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +0: [2023-02-05 02:34:51,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:51,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:51,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:51,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:51,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:51,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:51,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:51,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +7: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +0: [2023-02-05 02:34:51,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:51,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +2: [2023-02-05 02:34:51,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +4: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:51,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +3: [2023-02-05 02:34:51,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +1: [2023-02-05 02:34:51,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +5: [2023-02-05 02:34:51,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +3: [2023-02-05 02:34:51,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:51,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +1: [2023-02-05 02:34:51,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:51,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +2: [2023-02-05 02:34:51,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +7: [2023-02-05 02:34:51,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +5: [2023-02-05 02:34:51,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +4: [2023-02-05 02:34:51,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:51,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:51,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:51,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt... +6: [2023-02-05 02:34:51,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:51,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:51,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:52,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:52,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt. +6: [2023-02-05 02:34:52,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +4: [2023-02-05 02:34:52,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +3: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +1: [2023-02-05 02:34:52,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +0: [2023-02-05 02:34:52,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +7: [2023-02-05 02:34:52,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +3: [2023-02-05 02:34:52,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +7: [2023-02-05 02:34:52,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +5: [2023-02-05 02:34:52,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +7: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +7: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +4: [2023-02-05 02:34:52,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +4: [2023-02-05 02:34:52,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +2: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +6: [2023-02-05 02:34:52,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +5: [2023-02-05 02:34:52,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt... +5: [2023-02-05 02:34:52,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +2: [2023-02-05 02:34:52,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +5: [2023-02-05 02:34:52,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +5: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +2: [2023-02-05 02:34:52,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +4: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-02-05 02:34:52,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +0: [2023-02-05 02:34:52,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +0: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +0: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +0: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +1: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +0: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +1: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +3: [2023-02-05 02:34:52,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +3: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +3: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +1: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +1: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-02-05 02:34:52,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +1: [2023-02-05 02:34:52,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +2: [2023-02-05 02:34:52,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +1: [2023-02-05 02:34:52,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +1: [2023-02-05 02:34:52,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +1: [2023-02-05 02:34:52,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt. +6: [2023-02-05 02:34:52,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-02-05 02:34:52,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt... +6: [2023-02-05 02:34:52,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt. +6: [2023-02-05 02:34:52,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +0: [2023-02-05 02:34:52,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:52,684] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +0: [2023-02-05 02:34:52,690] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +5: [2023-02-05 02:34:52,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,790] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +0: [2023-02-05 02:34:52,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:52,794] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +5: [2023-02-05 02:34:52,796] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +0: [2023-02-05 02:34:52,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:52,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +0: [2023-02-05 02:34:52,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:52,800] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +0: [2023-02-05 02:34:52,800] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +0: [2023-02-05 02:34:52,806] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +0: [2023-02-05 02:34:52,807] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +5: [2023-02-05 02:34:52,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,818] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +5: [2023-02-05 02:34:52,824] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +7: [2023-02-05 02:34:52,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:52,840] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +6: [2023-02-05 02:34:52,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,844] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +7: [2023-02-05 02:34:52,847] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +6: [2023-02-05 02:34:52,850] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +7: [2023-02-05 02:34:52,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:52,856] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +2: [2023-02-05 02:34:52,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-05 02:34:52,857] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +7: [2023-02-05 02:34:52,862] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +2: [2023-02-05 02:34:52,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-02-05 02:34:52,863] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +2: [2023-02-05 02:34:52,864] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +2: [2023-02-05 02:34:52,869] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +5: [2023-02-05 02:34:52,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,870] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +5: [2023-02-05 02:34:52,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,876] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +5: [2023-02-05 02:34:52,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,877] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +5: [2023-02-05 02:34:52,878] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +5: [2023-02-05 02:34:52,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,883] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +5: [2023-02-05 02:34:52,883] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +5: [2023-02-05 02:34:52,885] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +5: [2023-02-05 02:34:52,890] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +6: [2023-02-05 02:34:52,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,893] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +4: [2023-02-05 02:34:52,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:52,896] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +5: [2023-02-05 02:34:52,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,901] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +2: [2023-02-05 02:34:52,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,901] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +2: [2023-02-05 02:34:52,901] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +4: [2023-02-05 02:34:52,903] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +5: [2023-02-05 02:34:52,907] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +2: [2023-02-05 02:34:52,907] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +6: [2023-02-05 02:34:52,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,913] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +6: [2023-02-05 02:34:52,920] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +7: [2023-02-05 02:34:52,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:52,930] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +7: [2023-02-05 02:34:52,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:52,936] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +7: [2023-02-05 02:34:52,938] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +5: [2023-02-05 02:34:52,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +5: [2023-02-05 02:34:52,943] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +7: [2023-02-05 02:34:52,944] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +0: [2023-02-05 02:34:52,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:52,948] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +5: [2023-02-05 02:34:52,951] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +0: [2023-02-05 02:34:52,953] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +6: [2023-02-05 02:34:52,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,955] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +6: [2023-02-05 02:34:52,962] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +6: [2023-02-05 02:34:52,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,965] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +7: [2023-02-05 02:34:52,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:52,968] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +6: [2023-02-05 02:34:52,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,971] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +6: [2023-02-05 02:34:52,973] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +6: [2023-02-05 02:34:52,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:52,974] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +7: [2023-02-05 02:34:52,975] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +6: [2023-02-05 02:34:52,978] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +6: [2023-02-05 02:34:52,982] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +0: [2023-02-05 02:34:53,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:53,005] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +6: [2023-02-05 02:34:53,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-05 02:34:53,012] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +0: [2023-02-05 02:34:53,013] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +0: [2023-02-05 02:34:53,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:53,016] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +4: [2023-02-05 02:34:53,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:53,018] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +6: [2023-02-05 02:34:53,020] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +0: [2023-02-05 02:34:53,023] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +4: [2023-02-05 02:34:53,023] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +7: [2023-02-05 02:34:53,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:53,034] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +7: [2023-02-05 02:34:53,041] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +4: [2023-02-05 02:34:53,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:53,051] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +0: [2023-02-05 02:34:53,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-05 02:34:53,057] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +4: [2023-02-05 02:34:53,058] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +0: [2023-02-05 02:34:53,064] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +7: [2023-02-05 02:34:53,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:53,069] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +3: [2023-02-05 02:34:53,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,070] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +3: [2023-02-05 02:34:53,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,073] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +7: [2023-02-05 02:34:53,076] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +3: [2023-02-05 02:34:53,077] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +3: [2023-02-05 02:34:53,080] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +4: [2023-02-05 02:34:53,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:53,084] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +7: [2023-02-05 02:34:53,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-05 02:34:53,089] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +4: [2023-02-05 02:34:53,091] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +7: [2023-02-05 02:34:53,098] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +4: [2023-02-05 02:34:53,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:53,131] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +4: [2023-02-05 02:34:53,138] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +3: [2023-02-05 02:34:53,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,141] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +3: [2023-02-05 02:34:53,147] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +3: [2023-02-05 02:34:53,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,153] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +1: [2023-02-05 02:34:53,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,157] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +3: [2023-02-05 02:34:53,160] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +2: [2023-02-05 02:34:53,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,163] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +2: [2023-02-05 02:34:53,163] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +2: [2023-02-05 02:34:53,169] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +4: [2023-02-05 02:34:53,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:53,186] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +4: [2023-02-05 02:34:53,193] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +2: [2023-02-05 02:34:53,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +2: [2023-02-05 02:34:53,201] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +3: [2023-02-05 02:34:53,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,203] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +3: [2023-02-05 02:34:53,203] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +2: [2023-02-05 02:34:53,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-05 02:34:53,206] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +2: [2023-02-05 02:34:53,208] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +2: [2023-02-05 02:34:53,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,210] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +2: [2023-02-05 02:34:53,210] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +3: [2023-02-05 02:34:53,210] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +1: [2023-02-05 02:34:53,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,212] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +2: [2023-02-05 02:34:53,213] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +2: [2023-02-05 02:34:53,217] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +1: [2023-02-05 02:34:53,221] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +4: [2023-02-05 02:34:53,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:53,243] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +1: [2023-02-05 02:34:53,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,245] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +4: [2023-02-05 02:34:53,251] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +1: [2023-02-05 02:34:53,252] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +3: [2023-02-05 02:34:53,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,257] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +3: [2023-02-05 02:34:53,264] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +1: [2023-02-05 02:34:53,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,280] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +3: [2023-02-05 02:34:53,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +3: [2023-02-05 02:34:53,280] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +1: [2023-02-05 02:34:53,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,285] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +1: [2023-02-05 02:34:53,287] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +3: [2023-02-05 02:34:53,288] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +1: [2023-02-05 02:34:53,293] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +2: [2023-02-05 02:34:53,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-05 02:34:53,334] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +2: [2023-02-05 02:34:53,341] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +4: [2023-02-05 02:34:53,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-05 02:34:53,358] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +4: [2023-02-05 02:34:53,367] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +1: [2023-02-05 02:34:53,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,377] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +1: [2023-02-05 02:34:53,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,378] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +1: [2023-02-05 02:34:53,384] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +1: [2023-02-05 02:34:53,385] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +1: [2023-02-05 02:34:53,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-02-05 02:34:53,413] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +1: [2023-02-05 02:34:53,421] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +0: successfully loaded checkpoint from checkpoints_1b11b51b5 at iteration 0 +7: time (ms) | load-checkpoint: 9967.06 +0: estimated model parameters: 1.096338432 +0: estimated model parameters without embeddings: 1.002523648 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-05 02:34:54 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.030174 seconds +0: number of documents: 3133972 +0: > dataset split: +0: train: +0: document indices in [0, 3133972) total of 3133972 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.102 seconds +0: total number of samples: 731002 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.064785 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.062 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-05 02:35:08 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 27833.47 | train/valid/test-data-iterators-setup: 13724.58 +0: [after training is done] datetime: 2023-02-05 02:35:08 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 3.680017E+00 | lm loss PPL: 3.964708E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 2809896: Sun Feb 5 02:36:06 EET 2023 diff --git a/1b11b51b5/sbatch_1b11b51b5.sh b/1b11b51b5/sbatch_1b11b51b5.sh new file mode 100755 index 0000000000000000000000000000000000000000..eeff5a3c31498cc1ceed10184a6d627107d3b1c8 --- /dev/null +++ b/1b11b51b5/sbatch_1b11b51b5.sh @@ -0,0 +1,166 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=1b11b51b5 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train1b5.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_1143M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 1516071000 +# -> Samples: 740269 +TRAIN_SAMPLES=740_269 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 7403 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --loss-scale 12 \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/1b11b51b5/sbatch_1b11b51b5val.sh b/1b11b51b5/sbatch_1b11b51b5val.sh new file mode 100644 index 0000000000000000000000000000000000000000..fca3d6143ac9b92abe4762b1aa2ec51af1662804 --- /dev/null +++ b/1b11b51b5/sbatch_1b11b51b5val.sh @@ -0,0 +1,171 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 12:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=1b11b51b5val +VARIANT_CKPT=1b11b51b5 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +# DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train1b5.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + + + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_1143M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 1516071000 +# -> Samples: 740269 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --no-load-optim \ + --reset-progress \ + --override-lr-scheduler \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --loss-scale 12 \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-only true \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/1b11b51b5/tensorboard_1b11b51b5/events.out.tfevents.1675935717.nid007109.102960.0 b/1b11b51b5/tensorboard_1b11b51b5/events.out.tfevents.1675935717.nid007109.102960.0 new file mode 100644 index 0000000000000000000000000000000000000000..21a3d2c94472570df788e34bf01315cfd454be97 --- /dev/null +++ b/1b11b51b5/tensorboard_1b11b51b5/events.out.tfevents.1675935717.nid007109.102960.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f18bfcdc27003fb69c97bde45f74d9220dc18f77709f7b22b78012d9192a175 +size 21226 diff --git a/1b11b51b5/tensorboard_1b11b51b5_v2/events.out.tfevents.1675526547.nid006152.12571.0 b/1b11b51b5/tensorboard_1b11b51b5_v2/events.out.tfevents.1675526547.nid006152.12571.0 new file mode 100644 index 0000000000000000000000000000000000000000..e6bff96d7c6202459376c40092b11b3f656e3831 --- /dev/null +++ b/1b11b51b5/tensorboard_1b11b51b5_v2/events.out.tfevents.1675526547.nid006152.12571.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1176047a471d2c29e34b6e8adef9a33f983d88b6a48e4ad1f676f52a41a5e536 +size 5153654 diff --git a/1b11b51b5/tensorboard_1b11b51b5val/events.out.tfevents.1675557013.nid005014.8700.0 b/1b11b51b5/tensorboard_1b11b51b5val/events.out.tfevents.1675557013.nid005014.8700.0 new file mode 100644 index 0000000000000000000000000000000000000000..15395b2862914d105e46bb9a9dafddfcfd39378d --- /dev/null +++ b/1b11b51b5/tensorboard_1b11b51b5val/events.out.tfevents.1675557013.nid005014.8700.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8092008c2872ccb5889d083657c0156331641261ad69c75585cd34c79e6da026 +size 980 diff --git a/1b11b51b5/tensorboard_1b11b51b5val/events.out.tfevents.1675557135.nid005024.72200.0 b/1b11b51b5/tensorboard_1b11b51b5val/events.out.tfevents.1675557135.nid005024.72200.0 new file mode 100644 index 0000000000000000000000000000000000000000..323ad1c8f66cc7ea78e9aac2dd56c676eea6d32f --- /dev/null +++ b/1b11b51b5/tensorboard_1b11b51b5val/events.out.tfevents.1675557135.nid005024.72200.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d00357a5ace72dfeeb76af3bf3d51a8c6737675ddbedb5710f1ea31e94c84a12 +size 980 diff --git a/1b11b51b5/tensorboard_1b11b51b5val_v2/events.out.tfevents.1675976562.nid005011.103591.0 b/1b11b51b5/tensorboard_1b11b51b5val_v2/events.out.tfevents.1675976562.nid005011.103591.0 new file mode 100644 index 0000000000000000000000000000000000000000..7ca9483418eb97301e34dc4c56cc858534958df9 --- /dev/null +++ b/1b11b51b5/tensorboard_1b11b51b5val_v2/events.out.tfevents.1675976562.nid005011.103591.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5146faf56858982225f67ddd22b3617bb9c72873360408a981ae2a79760aa3aa +size 980 diff --git a/1b11b51b5/transformers/config.json b/1b11b51b5/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7120acb75187622c061f0425674b658dd9155502 --- /dev/null +++ b/1b11b51b5/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50304, "n_positions": 2048, "n_embd": 1792, "n_layer": 26, "n_head": 14, "n_inner": 7168, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/1b11b51b5/transformers/pytorch_model.bin b/1b11b51b5/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..14069144df9481de739a4432bf2f70dec2c57153 --- /dev/null +++ b/1b11b51b5/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efd74db5f2f3da7cc1b66c05c81c5819ce45e8f6df1764ff772ca6d02a1e557e +size 2410895501 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17b5416ad58bdceabec9f3ef6828b3d3901f27e4 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdc6b4b383f0f9da55e52c188db5c4cd2d41fa319a5b2db5d83b385b28de1ff1 +size 116013719 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e27a6aad22358a062f57ba734e8046e49fdbe837 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b682806ae837af13f44dd3b932a6f9a3b312a9898be20607dddda3b6113b0cad +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1e451c863386bbd5121fc50eadeb112e43820ef --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dc67339c8bc71129f127909a2b35393287dd9f6790c2aac9260233ef3f4cfc8 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ac27eb75781d262f4b761c5bfecd89aa7e37acb --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24041a4d29a2cc6ab6830fbb512591e5fbdecbd0da0b16461b7584ec18df076b +size 116013602 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2891dbf787b481a852d897765f6a2d81d8b2206f --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d85160da98b3e777fc061caaa607d6b20cf63e4fdd78c7876b7b549e2976f11d +size 116013858 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37fcde0023d00937b822dd3e1ed272e904b3e374 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:659e1db972205b34112aa1aabacc98e0962571da730dbb6e25e8893edc60bcfa +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1d2c7823aa40042cf4fe4ef082ada8a15ec0224 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a02f9e1681b93e4550e96678d285714d838e7331a51509dbab9e2d871531fa9 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1c8d17a46db94c2ce36e3ce4a1691546d607ecf --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:209159c1156bf2c324b28fa342167b190236efb028699893c3753413e20ca749 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..982f88c3c49d5508811e670c2ca8d32423a6c03b --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c445337b7279d0ac36bfbc6f8d3f9d4ae4fb5cba3c1b0ea397be3162c578ac5 +size 116013602 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29442093eff9264b07b7be27342f1801ba8b2876 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f2e9fdd7fe334f225a2dd28b4969fe77a59032f817df29569f17be2ad9637e1 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70d4190c714107bbe3588ded2beef96b2ef12ef0 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4f6d078e9b9ff6776cc75b3c92417cdf2400dfac1ae441b5d63ec5606f690b +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bab3765fd2dd4b2d58fbaded6deb8bc6b5f2e382 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb0d3e5e08c440baa37002dc72f57a5e67e2163fb47aca0c9e1d0985df30c4a +size 116013719 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08a89c1ae20cbed1f4992a15e2cb0b36fb73007c --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb4b9214cd7f63d9b055a0d4185c3d17f2c70564c83f388b7b197f1e42ed7f3c +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6aa22e6be87539d1ef9318cff59765c867bd7490 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d85c9091ca653fd7ba78baadddfc88ec660292d914d990ccc4e190ba4d72f0fc +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..239a0bcec315999b967cd92c8b3c7dfa6fef3b7a --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc9d53e376c278bf0cbca51165efa5c192d7097115256225ef15537bc5e9ff21 +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b80a609786db5a2f77144b2a070084fc9764c852 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9537b0217ea3401558e7a7f3790dd6d903d9d6d2e11a44f1805be7cdb1f52eef +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65b6dc06b1665d66b110a778f14b70f56b8165f4 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c0ef193fdd987cd239376ad5c38fc9e514ddef3d12136660ba11cf05de64750 +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..acb5e04800e1dec9ba3667eb312fd25df58ea258 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88678319a2174662014ea2674e3d3a8fd4f4cb89eec445d22bd0db41504d27ed +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a283c3b24f6765c242ba9d35a24b87aa7de63c3c --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8fa57971770d90634c4f61bb25ef6a67024f3a4f5228eaa59b5246a977072e4 +size 116013858 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8aa01a7d66e157dd1b5298e591b60ca8473107a --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2931a6f3e33691cb3711e5dbfb3cba03ac86c72fd391cfb8e6c16c265839c6c6 +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b58ee00140c013df2804a52dce037d54693b624 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cd5aaa1aefbf5e2ee693fed6d77727d5ad697cf37626506b695bbda7d75f7b0 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b106fcc57958fcbf67b880e342ffd5380e4f0272 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:222eadeff06444c78aca0a2c5532bb478f624bf83ffc8d035ddadd7b6efe112c +size 116013602 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec8063f31694f37ccadf0baec51ce04021cf913d --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dad83890694fea6f0b02d4ae768aacddb40eef98627eeadf27d1b4e616a9dfc +size 116013655 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77b8895e225767b57cfdc37a9d36c2afcd63dfb0 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:661514a7d943a35cd903fbfa82722378c71866e0036665aff6e8806616fe40c7 +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9bb4869917618d9f262012e42701e5040c0810c --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cdfc60abf09e7fd87c5e2bac5622be1f13117b11c6e9ce336aaee56d3e94788 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb7ad8cdd3ccf216c1b2595455ebe37d18f8abbf --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4653ee279882aa42288f74c2604e17444f16039f73022821a0b90942eda7a27 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bd4768021faef71c41e1dd0080ae8178ffc23c2 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d0d720b50d8086361e11180a289a34988019d1e90195b5ce7ebde3f7f89316a +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5eed5e938f8dc602ca61ec2ff4312bc0876b8f4f --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37b818890efd12602289460ff15287012b9d00cf5633fb79bb255b337d1702d +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..baa0938aa2b4a5bd5e20c7390631e456c3e5f1d8 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5064a08fe03dbfefb015dcc800dbc7d11e44f80a07f6d1dbefe81b75410147de +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4960ba9c162381517510ba4e27313caf2981760 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cca10b44e6b9ad42749b30eddaf1613474d4a17fad6580f7a5c827b10ab8bda +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b3ccc281f89681f28f5298d9ef527bdbfc687b4 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac27cc216e6a74ef492be1fdc2e48ea3bb7b6afb6608b660948cafaa6ce3f2a4 +size 116013858 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ad46bbadec6ef200a9133ef927daac4551a17e8 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2359b0c205dc1dc31ea7a45437928d118deb9c91050bdf34d8b712b3153e151b +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60a1306f9560a8c9899e15b71db980b86946fa4e --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd9ad35f9f3f139dda60de68c2725ae533cbf6f5959481e0e9e13153e47d0ee4 +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3283a4219cbd36e0cbdf6bc2a1877d421c33e9c0 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81df97e4c2ae442b6974e33d2b844941141d3f248eca89eda2275c1c7ff9bc22 +size 116013719 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71a07ceaab011997f86667b6b3b0522863c31926 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5f72a70beaeae0fa1777220b95456116af9278de9d55c68d29fd318a1e6074d +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1986c7bb5ce0efb1c055cac94598dc7d62a81f8a --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c046c01819f47206073ffe37fa61a4a5dc077f472598daba8703bb27b544de60 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f5e8e424d100cff2bab0c070cd055cf6b558f76 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba58d36908aab603795eb33e25b88e90a1bc167701587f5296dc88818b49071a +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c0cb98dfbe3eea3aa4b1ee859a294a4542a489f --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02212a94d5c784b53bd1321ba702ffcc32202b579b7635ee48bc7c588ee60039 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01289a7f1b4b7a5d6d021b147bdc59c3e093013b --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a852192fbf8c2e9dcce28316064b94de3da57db39e8db7bdb707b6dade92069e +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aad345f15ccdebb397ad1b98d60a0aea964f52a3 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45b899de174af22743ca2d96691fd8877d46ad3aa09e89566182aae5c1fdf863 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b676790086d8dfeb4d690b60af609b199af13a83 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79d88fe53f594363774e4be41171ecc6b28d118d0c3eb9307d789c04706d620e +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e382a22d4e2d5f03cc93b0a8b200320f997ef63 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4395dfebf476d2b72aa830099d7c8580d7f75774660f423a825764584c811d29 +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38ec4b822cb9031bba53a4d396f659fd8ed55e63 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a28b2b854f39098eea3c9edfaf380e5a5357a59a65be97a9dffdc3d26d8d99a +size 116013922 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7444a68877de00ddae2023d1b29ff487e862a031 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cda2fe4abb20c4a527eaca71cecc904f4298a515eac7bb44cd36dd73b77d40f9 +size 116013538 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fc538dc5ee26c9d6e25e21e880a0cbcc046e98e --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67a1e07a05776e785aa74e10207cd3cf3ab44bd3faf57ae3d415b7d0ad12e17c +size 116013719 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..943f806a6f643d849bb7762b3dbbd23fda2333ad --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bddf46c67c554f804ecb8e8e842941b2378207405b8e89599465c651e661410 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b998b903bc02801c3045ea40f6a9134951f18094 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9731a0bd6e3bc7c81044b03e400fca31e1e4d4297a916895a0787626b5206081 +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc453185b0f31797a296377b19a566eb2360d2ed --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b69be2ae8d4a470a935b4a74ff0d062fdd159a3505c75dedf09122b940507c0d +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aacb70f817065bc8de155e1544fb9caa1b224e70 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a71dbcc06b75eb6c31354f982093a47e8ff8a0fb5dddfecac4d852c236c8b908 +size 116013922 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45bc21a428d7bdd5cf309f18c26c2653719dbab8 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a6d28039af508b3022cafa19fadcbb244c737d7ee2b59e4a4a746351014d70 +size 116013602 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab8c7f96063329d9a4a20ea23e26216c23a848ef --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da771f89d864badfe0b9298f198eba754a1f9282a3fbb029459cb307081f22fd +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..574b08b0c2aaad5cb86e4d13fd62827a839f2d1c --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f712ed98fbd1a5aa4f28aa8841aa13773f16e541f65c114e0c4443e1ea824158 +size 116013602 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4267bf1ae1335afb920ec69b898385aefca819a --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac339b58b6a2ac71b99da76b0253cf24b0614b1eff0374d36b44e981f6c7bc75 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c50714d1a2e3f4eb9ad2802457223025d6b4064 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:447033b37d9efcfa7853f9b74fc4134b7fd9dc0e320006f56acca486768f8534 +size 116013858 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ce01d56a171a1dd0831f1d6a5f3cf5b9f827ffc --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2e861277b7bef4e20cdcb6803ea07dfb3925f51e7145c61b3eb7c08c357756 +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fca913767dfaa275cb0be226ab946e9b90082e7f --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5fabe3ea2a0ff40a9b5856c74673fe94076aa6e307ef1c33e528c94d9f47288 +size 116013655 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15e590fc0fc109088f88f57854a7b661bc667590 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79506d97ba6d740311b07cc5bd33645fe09b0f55f7d22b1060b09605f86d6826 +size 116013730 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51009632d88bec593b450e8aed3cb676d9696c1d --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bab53cd4baaf032b9475ffc57705f533be4648284c6c632162a14e3b82576fb +size 116013794 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06f7d8fd485b17442a0ae8f0b42d78255355b9dc --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ec2867e08c4d745fd90ff79055d7bfde1017384149433cdf42a7d511d1b2773 +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..519e4c226dcda1677cadc0267bcf28f85754342c --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:942cff786a411ae6aea13abdadd36ce0677acc9dc7b851f06c3f3aa6eb42d737 +size 116013666 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8192c083c8fc29e8228ea3c158af44b781e6a457 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bff73c9f75d671c6cad617a59b7b6f5890c3abfc04e34c142dd927fe35d3f8be +size 116013719 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abf58ccfd29b9d1ce507034a35b052d8503b5923 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:304bf08adc3ea957598f1353d89c378bb7a48415f3cb00957d2c58c2e49dc8c7 +size 116013719 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b064332f530b76f0eae07f046ae8f16787560d21 --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5925f2cfd8078f1d9d63c88f09db01db56fe782f7a1f630a6f1d4843c780164e +size 116013655 diff --git a/619m22b100m/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/619m22b100m/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bd34b51648e1a6fd938856ee3e7d48dd33a68ab --- /dev/null +++ b/619m22b100m/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28d83609e06274c000470a6ffd6eb5685d0efe77da94f7481435f5cf00cd5327 +size 116013591 diff --git a/619m22b100m/global_step41007/layer_01-model_00-model_states.pt b/619m22b100m/global_step41007/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36c23f9a181ea9e514a8730e44903befa7c2355f --- /dev/null +++ b/619m22b100m/global_step41007/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e6a44d7219be5d2f7736c8b82f259ef699c2c18e203e74601b46e48f3089513 +size 160826627 diff --git a/619m22b100m/global_step41007/layer_03-model_00-model_states.pt b/619m22b100m/global_step41007/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f32f9c69e2d10d4616908b7fa71484e0a53fec5 --- /dev/null +++ b/619m22b100m/global_step41007/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17606d3cc29e1727f1219b647959763d7c64662daa52c01ce84dd1105c856d1 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_04-model_00-model_states.pt b/619m22b100m/global_step41007/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..edc82bf2c0e0c8ff46f9143f6d39f3d58f94a9b7 --- /dev/null +++ b/619m22b100m/global_step41007/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b46d3d488a445cccfe83d7fef5e56e5ec1727b7758405c3599bdb939b50a6284 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_05-model_00-model_states.pt b/619m22b100m/global_step41007/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7f3352b9cfea69d787c4014306af3de0bd790c6 --- /dev/null +++ b/619m22b100m/global_step41007/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d8be9429f1b9c37e2490c4e55b66e748acbbb2d39df3a2a151035e41b45bf8a +size 56667395 diff --git a/619m22b100m/global_step41007/layer_06-model_00-model_states.pt b/619m22b100m/global_step41007/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc103f9401395368b627778eebaf646f8b23eff0 --- /dev/null +++ b/619m22b100m/global_step41007/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3660c9d79bf4135951f42bb8e6794c17c53aa77b45a88f82ebdb392b326e9d0 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_07-model_00-model_states.pt b/619m22b100m/global_step41007/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..601e822a210af21b3ed77133e60786e88a297386 --- /dev/null +++ b/619m22b100m/global_step41007/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01b7bfcae3b569717c5c5b16035874761e9328ec7f0474fc994003639b57ab9f +size 56667395 diff --git a/619m22b100m/global_step41007/layer_08-model_00-model_states.pt b/619m22b100m/global_step41007/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28856702df2c2f053d3d6b496e468ec8a8782182 --- /dev/null +++ b/619m22b100m/global_step41007/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cb0d879acd53da34cfcbee78a2fcff685d6c3f4fec3b83cb86b1eb2f1280e5a +size 56667395 diff --git a/619m22b100m/global_step41007/layer_09-model_00-model_states.pt b/619m22b100m/global_step41007/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e5896e6c3a85f89e131cec88a07bf26a79fd2a3 --- /dev/null +++ b/619m22b100m/global_step41007/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57fd41b8e409111da50217d50b25c8a3c932d98955e89e33e634af4779a35d30 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_10-model_00-model_states.pt b/619m22b100m/global_step41007/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0892a4a4cc11866670ae77d66418bc54238a49b3 --- /dev/null +++ b/619m22b100m/global_step41007/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:607278bcf9267389fb4cd036cf37afa7a8413330575245fc7b6064c63660e008 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_11-model_00-model_states.pt b/619m22b100m/global_step41007/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbe62868239f202772ac86b85543422b598927e6 --- /dev/null +++ b/619m22b100m/global_step41007/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d78abe394d26f0d302730f8d8623126dcd02695d3311779bd7250526232d13 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_12-model_00-model_states.pt b/619m22b100m/global_step41007/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c86d8e8226555313b3bdcf46a9576e308a8cd2dd --- /dev/null +++ b/619m22b100m/global_step41007/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5505a76e433143b7202a1f173dfabcf933e9124a48460cb3b6ea3672463de9c4 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_13-model_00-model_states.pt b/619m22b100m/global_step41007/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ebba21c0d544317d25c0ed6b2fb46c49e067a4b --- /dev/null +++ b/619m22b100m/global_step41007/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:554f25f62223f33aceddd9297cdb66e480b66a9c26fd807bbe65084e8488dbc7 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_14-model_00-model_states.pt b/619m22b100m/global_step41007/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..599dc15e3b36f402bd7c463f90848d0e167fbd19 --- /dev/null +++ b/619m22b100m/global_step41007/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57129ac1de1ff2970ccf7fe2af5410e9a9d057f5074faa8a8e31521848119643 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_15-model_00-model_states.pt b/619m22b100m/global_step41007/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d230bb47205637a63747c09de10d8255fec2a458 --- /dev/null +++ b/619m22b100m/global_step41007/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1faa9b8abe808964606d4614dff9efdab7dda9a8c981a0486c8b73146e0f6b1c +size 56667395 diff --git a/619m22b100m/global_step41007/layer_16-model_00-model_states.pt b/619m22b100m/global_step41007/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf2044854c10ad0e0daca382796ca955ccefd99c --- /dev/null +++ b/619m22b100m/global_step41007/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30379beb013c00802a23fce5e40f41c177ef2e90dbbd7703170c59fce8b856b9 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_17-model_00-model_states.pt b/619m22b100m/global_step41007/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12c23da9433e39a83a02f21ef5c6c26a4d90125d --- /dev/null +++ b/619m22b100m/global_step41007/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08a704944b14b5d187ff252a16ad645b3a434de9f4e97fbc6dbbe845c2896813 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_18-model_00-model_states.pt b/619m22b100m/global_step41007/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9736511c76eb3aac4fc8abc85e7efdf0a7df016d --- /dev/null +++ b/619m22b100m/global_step41007/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:175be1ad4bb822bef79ba5fa72e7287cc5e9e03a4039f68bad603429605e3e1a +size 56667395 diff --git a/619m22b100m/global_step41007/layer_19-model_00-model_states.pt b/619m22b100m/global_step41007/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05dd57534282b158f03d5b29dadd2e735dceb050 --- /dev/null +++ b/619m22b100m/global_step41007/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3df3cf6b5d801973e12ba48b4b6737cf83e0560a70c97ef95476f87ec7395ff4 +size 56667395 diff --git a/619m22b100m/global_step41007/layer_20-model_00-model_states.pt b/619m22b100m/global_step41007/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23c255c24d213f57cdb456bfab8b12bf930874ec --- /dev/null +++ b/619m22b100m/global_step41007/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1103ef365f7d452a7d5034cb38243ad7db1ef21b1f3ed05959257cff7d7912d +size 56667395 diff --git a/619m22b100m/global_step41007/layer_21-model_00-model_states.pt b/619m22b100m/global_step41007/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0587c317093efd564539efa1a97aa415d2967a00 --- /dev/null +++ b/619m22b100m/global_step41007/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b92f8b2af964d523f000090dc0755a4baf59d628d5344a6b5b0f7030a9aa3bc +size 56667395 diff --git a/619m22b100m/global_step41007/layer_23-model_00-model_states.pt b/619m22b100m/global_step41007/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07436559f4ce505f4f374ee8e18416dd67fa6ee3 --- /dev/null +++ b/619m22b100m/global_step41007/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:531506ff8aea53ee97bd3c757dd1f8644a9785c1e0c806340d6659e8280eb670 +size 7363 diff --git a/619m22b100m/global_step41007/mp_rank_00_model_states.pt b/619m22b100m/global_step41007/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b8fd8069440046908ee29dabcc9b01abe5f0a50 --- /dev/null +++ b/619m22b100m/global_step41007/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aad8c24869790503daa029f529995e7742a40ea3358ed2ab3ca15a8dc4803a13 +size 38515 diff --git a/619m22b100m/logs/3168317.err b/619m22b100m/logs/3168317.err new file mode 100644 index 0000000000000000000000000000000000000000..0c4c9b12676c20ee8f50c025037a243770dc18e6 --- /dev/null +++ b/619m22b100m/logs/3168317.err @@ -0,0 +1,1121 @@ +5: 2023-02-28 08:59:53.726747: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 08:59:53.726757: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 08:59:53.726753: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 08:59:53.726757: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 08:59:53.726767: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 08:59:53.726766: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 08:59:53.726768: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-28 08:59:53.726759: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730964: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730979: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730979: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730974: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730971: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730969: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730967: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-28 08:59:53.730969: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731624: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731628: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731637: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731635: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731640: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731629: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731621: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-28 08:59:53.731630: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732280: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732280: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732296: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732294: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732286: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732292: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732287: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-28 08:59:53.732307: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732411: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732416: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732412: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732422: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732425: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732413: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732417: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:53.732431: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780027: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780026: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780034: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780034: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780027: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780026: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780028: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-28 08:59:53.780025: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 08:59:53.780289: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 08:59:53.780301: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 08:59:53.780295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 08:59:53.780294: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 08:59:53.780296: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780352: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780352: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780356: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: 2023-02-28 08:59:53.780287: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 08:59:53.780299: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-28 08:59:53.780287: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780366: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780369: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780372: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780364: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-28 08:59:53.780359: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-28 08:59:55.336130: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336140: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336141: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336144: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:55.336364: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 08:59:55.336366: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 08:59:55.336372: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 08:59:55.336374: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 08:59:55.336376: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 08:59:55.336373: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 08:59:55.336379: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-28 08:59:55.336384: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372183: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372190: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:55.372766: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372765: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372772: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372778: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372775: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372779: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372779: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-28 08:59:55.372785: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:55.373661: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373667: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373670: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373669: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373670: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373675: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-28 08:59:55.373677: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.419731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.419745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.419746: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.419755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.419758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.419759: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.419762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.419774: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:55.420273: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.420283: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.420285: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.420288: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.420293: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.420296: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.420303: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-28 08:59:55.420305: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420190: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:55.420708: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420706: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420713: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420714: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420715: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420721: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420722: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:55.420732: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.446974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.446977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.446985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.446989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.446992: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.446985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.446992: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.447004: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:55.447406: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.447408: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.447410: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.447412: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.447415: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.447415: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.447420: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-28 08:59:55.447426: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448045: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448044: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:55.448489: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448491: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448495: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448496: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448499: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448499: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448501: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-28 08:59:55.448504: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.455850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.455861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.455857: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.455863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.455869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.455865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.455872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.455868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:55.456282: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.456284: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.456286: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.456290: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.456291: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.456292: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.456293: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-28 08:59:55.456294: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-28 08:59:59.683049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-28 08:59:59.683100: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.683046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: 2023-02-28 08:59:59.683213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-28 08:59:59.683097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.683058: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.683104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.683216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.683054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.683108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.683222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.683063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.683111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.683221: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.683058: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.683114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.683228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.683061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.683107: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.683223: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.683067: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.683107: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.683226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.683219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684806: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 08:59:59.684806: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 08:59:59.684801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684806: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-28 08:59:59.684815: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 08:59:59.684823: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 08:59:59.684825: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 08:59:59.684822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 08:59:59.684824: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-28 08:59:59.684826: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 08:59:59.685043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-28 08:59:59.685166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685058: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-28 08:59:59.685059: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 08:59:59.685063: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685058: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-28 08:59:59.685180: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685183: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685067: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 08:59:59.685068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685178: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-28 08:59:59.685068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-28 08:59:59.685074: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-28 08:59:59.685182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-28 08:59:59.685110: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.685182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.685186: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.685186: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.685189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-28 08:59:59.685200: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685204: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685206: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685206: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685206: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-28 08:59:59.685208: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 08:59:59.702966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:59.702977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:59.702979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:59.702975: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703162: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-28 08:59:59.702984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703172: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-28 08:59:59.702984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703176: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-28 08:59:59.702989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-28 08:59:59.703005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703176: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703183: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.703186: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703598: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703611: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703606: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.703615: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704280: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704294: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.704314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704662: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704675: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-28 08:59:59.704881: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704678: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704680: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.704684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: 2023-02-28 08:59:59.704880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.704981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: 2023-02-28 08:59:59.704883: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:59.704886: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-28 08:59:59.704895: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 08:59:59.704980: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: 2023-02-28 08:59:59.704889: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.704985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: 2023-02-28 08:59:59.704891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.704989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: 2023-02-28 08:59:59.704898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 08:59:59.704898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 08:59:59.704904: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.704999: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 08:59:59.704906: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 08:59:59.704906: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 08:59:59.704989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: 2023-02-28 08:59:59.704959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.704989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: 2023-02-28 08:59:59.704962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.705005: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 08:59:59.705008: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.705009: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 08:59:59.705011: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 08:59:59.705012: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 08:59:59.704974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-28 08:59:59.704977: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 08:59:59.705042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.705048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-28 08:59:59.705063: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-28 08:59:59.705067: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 08:59:59.705809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705813: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: 2023-02-28 08:59:59.705936: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705823: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705951: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 08:59:59.705820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: 2023-02-28 08:59:59.705948: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705830: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: 2023-02-28 08:59:59.705949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705832: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 08:59:59.705832: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705836: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-28 08:59:59.705839: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-28 08:59:59.705839: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: 2023-02-28 08:59:59.705949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-28 08:59:59.705884: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.705956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.705957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: 2023-02-28 08:59:59.706399: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706401: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706405: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706407: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706419: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 08:59:59.706420: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 08:59:59.706421: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 08:59:59.706426: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 08:59:59.706466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706467: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706484: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 08:59:59.706486: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 08:59:59.706489: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-28 08:59:59.706475: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-28 08:59:59.706504: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.705960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-28 08:59:59.705973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705974: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705977: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705978: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705978: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-28 08:59:59.705980: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_upper_triang_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +1: Successfully preprocessed all matching files. +1: Successfully preprocessed all matching files. +1: Successfully preprocessed all matching files. +2: Successfully preprocessed all matching files. +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: +3: +3: +3: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: +5: +5: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: +6: +6: +6: +6: +6: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +0: Building extension module utils... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module utils... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Loading extension module utils...Loading extension module utils... +0: +0: Loading extension module utils...Loading extension module utils...Loading extension module utils... +0: Loading extension module utils... +0: +0: +1: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +6: Loading extension module utils... +5: Loading extension module utils... +6: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +5: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils... +1: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +5: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +5: +5: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +5: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils...Loading extension module utils... +4: +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +4: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +4: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +2: +2: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +6: +6: Loading extension module utils...Loading extension module utils... +6: +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +6: +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +7: +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/619m22b100m/logs/3168317.out b/619m22b100m/logs/3168317.out new file mode 100644 index 0000000000000000000000000000000000000000..1f494b95a48fdbf2ec5837bea75b2a76e6ff7b8f --- /dev/null +++ b/619m22b100m/logs/3168317.out @@ -0,0 +1,6692 @@ +Model parameters: d_model 1536 ffw_size 6144 kv_size 128 n_heads 12 n_layers 19 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 19 --hidden-size 1536 --num-attention-heads 12 --kv-channels 128 --ffn-hidden-size 6144 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-619m22b100mval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --no-load-optim --reset-progress --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --tensorboard-dir tensorboard_619m22b100mval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_619m22b100m --load checkpoints_619m22b100m --train-weighted-split-paths-path train100m.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3168317.json --zero-stage 0 +START 3168317: Tue 28 Feb 2023 08:59:35 AM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 49.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 42.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 39.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 40.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 49.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 47.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 47.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 41.0c 101.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 46.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 56.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 45.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 45.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 45.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 42.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 48.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 38.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 48.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 43.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +6: Launching on nid005953 (6/8), master nid005947 port 9999, GPUs 8, CUDA: True +3: Launching on nid005950 (3/8), master nid005947 port 9999, GPUs 8, CUDA: True +1: Launching on nid005948 (1/8), master nid005947 port 9999, GPUs 8, CUDA: True +4: Launching on nid005951 (4/8), master nid005947 port 9999, GPUs 8, CUDA: True +7: Launching on nid005954 (7/8), master nid005947 port 9999, GPUs 8, CUDA: True +2: Launching on nid005949 (2/8), master nid005947 port 9999, GPUs 8, CUDA: True +5: Launching on nid005952 (5/8), master nid005947 port 9999, GPUs 8, CUDA: True +0: Launching on nid005947 (0/8), master nid005947 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/3168317.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... None +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 6144 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1536 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-619m22b100mval +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_619m22b100m +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... None +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 12 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 19 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_619m22b100m +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_619m22b100mval +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +7: > setting tensorboard ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-28 09:00:18,761] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.082 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 87 +0: ninja: no work to do. +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 63 +0: [1/1] c++ scaled_masked_softmax_hip.cuda.o scaled_masked_softmax_hip.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: [1/1] c++ layer_norm_hip_kernel.cuda.o layer_norm_cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so +0: >>> done with compiling and loading fused kernels. Compilation time: 17.734 seconds +0: time to initialize megatron (seconds): 23.515 +0: [after megatron is initialized] datetime: 2023-02-28 09:00:39 +0: building GPT model ... +0: [2023-02-28 09:00:39,648] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-28 09:00:39,649] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-28 09:00:39,649] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.55 GB, percent = 6.1% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-28 09:00:41,715] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=26 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: undo +0: 23: MixedFusedLayerNorm +0: 24: EmbeddingPipe +0: 25: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-28 09:00:41,948] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-28 09:00:41,949] [INFO] [utils.py:828:see_memory_usage] MA 1.16 GB Max_MA 1.16 GB CA 1.2 GB Max_CA 1 GB +0: [2023-02-28 09:00:41,949] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.59 GB, percent = 6.1% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-28 09:00:41,951] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-28 09:00:54,551] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-28 09:00:54,551] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-28 09:00:54,552] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-28 09:00:54,558] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-28 09:00:54,558] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-28 09:00:54,677] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-28 09:00:54,678] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.17 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-28 09:00:54,678] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.26 GB, percent = 6.2% +0: ninja: no work to do. +0: Time to load utils op: 0.14533305168151855 seconds +0: Time to load utils op: 0.10214662551879883 seconds +0: Time to load utils op: 0.0006480216979980469 seconds +0: Time to load utils op: 0.20504164695739746 secondsTime to load utils op: 0.20466041564941406 seconds +0: +0: Time to load utils op: 0.20439481735229492 seconds +0: Time to load utils op: 0.2044377326965332 secondsTime to load utils op: 0.20455241203308105 secondsTime to load utils op: 0.20485329627990723 seconds +0: +0: +2: Time to load utils op: 0.2111494541168213 seconds +2: Time to load utils op: 0.2111663818359375 seconds +2: Time to load utils op: 0.21114563941955566 seconds +2: Time to load utils op: 0.21120047569274902 seconds +2: Time to load utils op: 0.21115517616271973 seconds +2: Time to load utils op: 0.2114124298095703 secondsTime to load utils op: 0.21114420890808105 seconds +2: +2: Time to load utils op: 0.21114659309387207 seconds +1: Time to load utils op: 0.21368908882141113 seconds +1: Time to load utils op: 0.2129380702972412 secondsTime to load utils op: 0.2130448818206787 seconds +1: +1: Time to load utils op: 0.21370196342468262 secondsTime to load utils op: 0.21394801139831543 seconds +1: +1: Time to load utils op: 0.21273279190063477 seconds +1: Time to load utils op: 0.21314167976379395 seconds +1: Time to load utils op: 0.21341347694396973 seconds +3: Time to load utils op: 0.21143698692321777 seconds +3: Time to load utils op: 0.2114574909210205 seconds +3: Time to load utils op: 0.21146106719970703 seconds +3: Time to load utils op: 0.21146440505981445 secondsTime to load utils op: 0.21147632598876953 secondsTime to load utils op: 0.21146774291992188 seconds +3: +3: +3: Time to load utils op: 0.2114572525024414 seconds +3: Time to load utils op: 0.21146869659423828 seconds +4: Time to load utils op: 0.21128511428833008 seconds +4: Time to load utils op: 0.21132540702819824 seconds +4: Time to load utils op: 0.21130156517028809 seconds +4: Time to load utils op: 0.2113044261932373 seconds +4: Time to load utils op: 0.21132302284240723 seconds +4: Time to load utils op: 0.21136736869812012 secondsTime to load utils op: 0.21135878562927246 secondsTime to load utils op: 0.21135401725769043 seconds +4: +4: +5: Time to load utils op: 0.21029949188232422 secondsTime to load utils op: 0.21030735969543457 seconds +5: +5: Time to load utils op: 0.21031665802001953 secondsTime to load utils op: 0.21031570434570312 seconds +5: +5: Time to load utils op: 0.2103281021118164 seconds +5: Time to load utils op: 0.21033000946044922 secondsTime to load utils op: 0.2103428840637207 seconds +5: Time to load utils op: 0.21034717559814453 seconds +5: +0: Time to load utils op: 0.00046706199645996094 seconds +0: Time to load utils op: 0.0003910064697265625 seconds +6: Time to load utils op: 0.2117938995361328 secondsTime to load utils op: 0.21178150177001953 seconds +6: +6: Time to load utils op: 0.21181368827819824 seconds +6: Time to load utils op: 0.21180415153503418 secondsTime to load utils op: 0.21181392669677734 secondsTime to load utils op: 0.21181344985961914 seconds +6: Time to load utils op: 0.21182990074157715 seconds +6: +6: Time to load utils op: 0.2118215560913086 seconds +6: +0: Time to load utils op: 0.0003838539123535156 seconds +0: Time to load utils op: 0.00038933753967285156 seconds +0: Time to load utils op: 0.0003864765167236328 seconds +0: Time to load utils op: 0.0003972053527832031 seconds +7: Time to load utils op: 0.2112429141998291 seconds +7: Time to load utils op: 0.21124935150146484 seconds +7: Time to load utils op: 0.21124887466430664 secondsTime to load utils op: 0.21126365661621094 seconds +7: +7: Time to load utils op: 0.21126890182495117 secondsTime to load utils op: 0.21142864227294922 seconds +7: +7: Time to load utils op: 0.21133875846862793 seconds +7: Time to load utils op: 0.21135449409484863 seconds +1: Time to load utils op: 0.0004744529724121094 seconds +1: Time to load utils op: 0.00041556358337402344 seconds +1: Time to load utils op: 0.0003974437713623047 seconds +1: Time to load utils op: 0.00039124488830566406 seconds +1: Time to load utils op: 0.0004985332489013672 seconds +1: Time to load utils op: 0.0005390644073486328 seconds +1: Time to load utils op: 0.0005519390106201172 seconds +1: Time to load utils op: 0.0005242824554443359 seconds +5: Time to load utils op: 0.0009286403656005859 seconds +5: Time to load utils op: 0.000993967056274414 seconds +5: Time to load utils op: 0.0010242462158203125 seconds +5: Time to load utils op: 0.0012853145599365234 seconds +5: Time to load utils op: 0.0012776851654052734 seconds +5: Time to load utils op: 0.0013294219970703125 seconds +5: Time to load utils op: 0.0012919902801513672 seconds +5: Time to load utils op: 0.0013628005981445312 seconds +4: Time to load utils op: 0.00096893310546875 seconds +4: Time to load utils op: 0.0009398460388183594 seconds +2: Time to load utils op: 0.0010962486267089844 seconds +4: Time to load utils op: 0.0011687278747558594 seconds +4: Time to load utils op: 0.0012874603271484375 seconds +4: Time to load utils op: 0.0013082027435302734 seconds +4: Time to load utils op: 0.001203775405883789 seconds +2: Time to load utils op: 0.0012485980987548828 seconds +2: Time to load utils op: 0.0012650489807128906 seconds +4: Time to load utils op: 0.0012590885162353516 seconds +2: Time to load utils op: 0.0013115406036376953 seconds +4: Time to load utils op: 0.0013165473937988281 seconds +2: Time to load utils op: 0.0013849735260009766 seconds +2: Time to load utils op: 0.0014085769653320312 seconds +2: Time to load utils op: 0.001407623291015625 seconds +2: Time to load utils op: 0.0013756752014160156 seconds +3: Time to load utils op: 0.0009624958038330078 seconds +3: Time to load utils op: 0.0012979507446289062 seconds +3: Time to load utils op: 0.0013422966003417969 seconds +3: Time to load utils op: 0.0013823509216308594 seconds +3: Time to load utils op: 0.001371145248413086 seconds +6: Time to load utils op: 0.0009849071502685547 secondsTime to load utils op: 0.0009593963623046875 seconds +6: +3: Time to load utils op: 0.00142669677734375 seconds +3: Time to load utils op: 0.0014066696166992188 seconds +3: Time to load utils op: 0.0014357566833496094 seconds +6: Time to load utils op: 0.001085042953491211 seconds +6: Time to load utils op: 0.0013244152069091797 seconds +6: Time to load utils op: 0.0014276504516601562 secondsTime to load utils op: 0.0014483928680419922 secondsTime to load utils op: 0.0013544559478759766 seconds +6: +6: +6: Time to load utils op: 0.0013918876647949219 seconds +7: Time to load utils op: 0.00084686279296875 seconds +7: Time to load utils op: 0.0007312297821044922 seconds +7: Time to load utils op: 0.0007376670837402344 seconds +7: Time to load utils op: 0.0010421276092529297 seconds +7: Time to load utils op: 0.0012159347534179688 seconds +7: Time to load utils op: 0.000978708267211914 seconds +7: Time to load utils op: 0.0010056495666503906 seconds +7: Time to load utils op: 0.0010645389556884766 seconds +0: [2023-02-28 09:00:54,900] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-02-28 09:00:54,901] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.15 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-28 09:00:54,901] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,017] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-28 09:00:55,018] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-28 09:00:55,018] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,120] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-28 09:00:55,121] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-28 09:00:55,121] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,224] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-28 09:00:55,224] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 09:00:55,224] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,325] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-28 09:00:55,326] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 09:00:55,326] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,430] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-28 09:00:55,431] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 09:00:55,431] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,531] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-28 09:00:55,532] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 09:00:55,532] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,637] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-28 09:00:55,638] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 09:00:55,638] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,739] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-28 09:00:55,740] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-28 09:00:55,740] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 31.41 GB, percent = 6.2% +0: [2023-02-28 09:00:55,740] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-28 09:00:55,740] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-28 09:00:55,740] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-28 09:00:55,741] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-28 09:00:55,741] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-28 09:00:55,741] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-28 09:00:55,741] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-28 09:00:55,741] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-28 09:00:55,741] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-28 09:00:55,742] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-28 09:00:55,743] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-28 09:00:55,743] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.0004248619079589844 seconds +0: [2023-02-28 09:00:55,744] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-28 09:00:55,755] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=26 [0, 26) STAGE_PARAMS=618714624 (618.715M) TOTAL_PARAMS=618714624 (618.715M) UNIQUE_PARAMS=618714624 (618.715M) +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-28 09:00:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:56,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:56,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:56,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:56,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-28 09:00:56,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-28 09:00:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-28 09:00:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:56,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:56,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:56,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-28 09:00:56,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-28 09:00:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-28 09:00:56,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-28 09:00:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-28 09:00:56,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-28 09:00:56,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-28 09:00:56,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-28 09:00:56,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-28 09:00:56,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-28 09:00:56,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-28 09:00:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-28 09:00:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-28 09:00:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-28 09:00:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-28 09:00:56,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-28 09:00:56,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-28 09:00:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-28 09:00:56,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-28 09:00:56,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-28 09:00:56,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-28 09:00:56,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-28 09:00:56,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-28 09:00:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-28 09:00:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-28 09:00:56,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-28 09:00:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-28 09:00:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-28 09:00:56,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-28 09:00:56,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-28 09:00:56,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-28 09:00:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-28 09:00:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-28 09:00:56,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-28 09:00:56,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-28 09:00:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-28 09:00:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-28 09:00:56,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-28 09:00:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-28 09:00:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-28 09:00:56,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-28 09:00:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-28 09:00:56,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-28 09:00:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-28 09:00:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-28 09:00:56,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-28 09:00:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-28 09:00:56,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-28 09:00:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-28 09:00:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-28 09:00:56,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-28 09:00:56,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-28 09:00:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-28 09:00:56,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:56,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:56,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-28 09:00:56,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-28 09:00:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-28 09:00:56,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:56,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:56,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:56,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-28 09:00:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-28 09:00:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-28 09:00:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-28 09:00:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-28 09:00:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-28 09:00:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-28 09:00:57,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-28 09:00:57,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-28 09:00:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-28 09:00:57,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-28 09:00:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-28 09:00:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-28 09:00:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-28 09:00:57,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-28 09:00:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-28 09:00:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-28 09:00:57,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-28 09:00:57,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-28 09:00:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-28 09:00:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-28 09:00:57,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-28 09:00:57,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-28 09:00:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-28 09:00:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-28 09:00:57,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-28 09:00:57,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-28 09:00:57,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-28 09:00:57,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-28 09:00:57,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-28 09:00:57,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-28 09:00:57,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-28 09:00:57,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-28 09:00:57,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-28 09:00:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-28 09:00:57,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-28 09:00:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-28 09:00:57,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-28 09:00:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-28 09:00:57,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-28 09:00:57,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-28 09:00:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-28 09:00:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-28 09:00:57,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-28 09:00:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-28 09:00:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-28 09:00:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-28 09:00:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-28 09:00:57,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-28 09:00:57,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-28 09:00:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-28 09:00:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-28 09:00:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-28 09:00:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-28 09:00:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-28 09:00:57,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-28 09:00:57,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-28 09:00:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-28 09:00:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-28 09:00:57,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-28 09:00:57,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-28 09:00:57,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-28 09:00:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-28 09:00:57,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-28 09:00:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-28 09:00:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-28 09:00:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-28 09:00:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-28 09:00:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-28 09:00:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-28 09:00:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-28 09:00:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-28 09:00:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-28 09:00:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:57,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-28 09:00:57,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-28 09:00:57,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-28 09:00:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-28 09:00:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-28 09:00:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-28 09:00:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-28 09:00:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-28 09:00:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:57,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:57,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-28 09:00:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-28 09:00:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-28 09:00:57,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:57,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-28 09:00:57,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:57,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-28 09:00:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-28 09:00:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:58,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-28 09:00:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-28 09:00:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-28 09:00:58,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-28 09:00:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-28 09:00:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-28 09:00:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-28 09:00:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-28 09:00:58,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-28 09:00:58,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-28 09:00:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-28 09:00:58,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-28 09:00:58,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-28 09:00:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-28 09:00:58,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:58,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:58,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-28 09:00:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-28 09:00:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:58,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:58,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-28 09:00:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-28 09:00:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-28 09:00:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-28 09:00:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:59,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:59,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:59,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:59,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:59,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-28 09:00:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:59,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-28 09:00:59,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:59,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-28 09:00:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-28 09:00:59,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-28 09:00:59,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-28 09:00:59,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-28 09:00:59,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-28 09:00:59,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-28 09:00:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-28 09:00:59,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-28 09:00:59,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-28 09:00:59,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-28 09:00:59,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-28 09:00:59,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-28 09:00:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-28 09:00:59,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-28 09:00:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-28 09:00:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-28 09:00:59,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-28 09:00:59,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-28 09:00:59,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-28 09:00:59,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-28 09:00:59,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-28 09:00:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-28 09:00:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-28 09:00:59,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-28 09:00:59,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-28 09:00:59,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-28 09:00:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-28 09:00:59,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-28 09:00:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-28 09:00:59,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-28 09:00:59,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-28 09:00:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-28 09:00:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-28 09:00:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-28 09:00:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-28 09:00:59,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-28 09:00:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-28 09:00:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-28 09:00:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-28 09:00:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-28 09:00:59,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-28 09:00:59,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-28 09:00:59,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-28 09:00:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-28 09:00:59,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-28 09:00:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-28 09:00:59,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-28 09:00:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-28 09:00:59,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-28 09:00:59,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-28 09:00:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-28 09:00:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-28 09:00:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-28 09:00:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-28 09:00:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-28 09:00:59,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-28 09:00:59,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-28 09:00:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-28 09:00:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-28 09:00:59,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-28 09:00:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-28 09:00:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-28 09:00:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-28 09:00:59,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-28 09:00:59,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-28 09:00:59,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-28 09:00:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:00:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-28 09:00:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:00:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:00:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:00:59,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-28 09:00:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:00:59,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-28 09:00:59,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:00:59,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-28 09:00:59,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:00:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:00:59,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:00:59,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:00:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:00:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:00:59,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-28 09:00:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:00:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:00:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:00:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:00:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-28 09:00:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-28 09:00:59,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:00:59,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:00:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:00:59,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:00:59,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:01:00,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:01:00,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:01:00,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:01:00,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:01:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-28 09:01:00,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:01:00,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:01:00,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-28 09:01:00,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-28 09:01:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-28 09:01:00,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-28 09:01:00,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-28 09:01:00,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-28 09:01:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-28 09:01:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-28 09:01:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-28 09:01:00,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-28 09:01:00,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-28 09:01:00,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-28 09:01:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-28 09:01:00,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-28 09:01:00,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-28 09:01:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-28 09:01:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-28 09:01:00,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-28 09:01:00,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-28 09:01:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-28 09:01:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +5: [2023-02-28 09:01:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +5: [2023-02-28 09:01:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-02-28 09:01:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +5: [2023-02-28 09:01:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-28 09:01:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-02-28 09:01:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +5: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +6: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +6: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-28 09:01:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +6: [2023-02-28 09:01:00,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +4: [2023-02-28 09:01:00,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-28 09:01:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +0: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-28 09:01:00,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +1: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +2: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +0: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +1: [2023-02-28 09:01:00,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-28 09:01:00,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-28 09:01:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-28 09:01:00,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-28 09:01:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-28 09:01:00,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +0: [2023-02-28 09:01:00,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +0: [2023-02-28 09:01:00,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +0: [2023-02-28 09:01:00,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +7: [2023-02-28 09:01:00,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-28 09:01:00,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-28 09:01:00,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +3: [2023-02-28 09:01:00,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:00,578] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +3: [2023-02-28 09:01:00,582] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +4: [2023-02-28 09:01:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-28 09:01:00,592] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +4: [2023-02-28 09:01:00,596] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +7: [2023-02-28 09:01:00,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,596] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +3: [2023-02-28 09:01:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:00,599] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +7: [2023-02-28 09:01:00,601] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +3: [2023-02-28 09:01:00,603] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +4: [2023-02-28 09:01:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-28 09:01:00,604] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +6: [2023-02-28 09:01:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,604] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +6: [2023-02-28 09:01:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,607] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +6: [2023-02-28 09:01:00,608] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +4: [2023-02-28 09:01:00,608] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +5: [2023-02-28 09:01:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,613] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +5: [2023-02-28 09:01:00,613] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +5: [2023-02-28 09:01:00,617] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +6: [2023-02-28 09:01:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,624] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +2: [2023-02-28 09:01:00,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,625] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +6: [2023-02-28 09:01:00,629] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +2: [2023-02-28 09:01:00,629] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +0: [2023-02-28 09:01:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,638] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +0: [2023-02-28 09:01:00,642] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +5: [2023-02-28 09:01:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-28 09:01:00,650] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +5: [2023-02-28 09:01:00,654] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +1: [2023-02-28 09:01:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-28 09:01:00,658] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +7: [2023-02-28 09:01:00,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,659] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +4: [2023-02-28 09:01:00,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-02-28 09:01:00,660] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +6: [2023-02-28 09:01:00,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,660] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +1: [2023-02-28 09:01:00,662] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +7: [2023-02-28 09:01:00,662] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +5: [2023-02-28 09:01:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-28 09:01:00,664] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +6: [2023-02-28 09:01:00,665] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +4: [2023-02-28 09:01:00,665] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +5: [2023-02-28 09:01:00,668] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +3: [2023-02-28 09:01:00,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:00,669] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +5: [2023-02-28 09:01:00,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-02-28 09:01:00,670] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +4: [2023-02-28 09:01:00,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +4: [2023-02-28 09:01:00,670] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +3: [2023-02-28 09:01:00,673] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +5: [2023-02-28 09:01:00,674] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +4: [2023-02-28 09:01:00,675] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +1: [2023-02-28 09:01:00,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-28 09:01:00,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-28 09:01:00,676] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +1: [2023-02-28 09:01:00,676] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +2: [2023-02-28 09:01:00,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +1: [2023-02-28 09:01:00,680] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +1: [2023-02-28 09:01:00,681] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +2: [2023-02-28 09:01:00,681] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +4: [2023-02-28 09:01:00,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-28 09:01:00,686] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +3: [2023-02-28 09:01:00,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:00,689] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +4: [2023-02-28 09:01:00,691] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +3: [2023-02-28 09:01:00,693] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +7: [2023-02-28 09:01:00,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,695] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +3: [2023-02-28 09:01:00,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:00,698] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +6: [2023-02-28 09:01:00,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,699] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +0: [2023-02-28 09:01:00,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,699] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +0: [2023-02-28 09:01:00,700] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +6: [2023-02-28 09:01:00,702] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +3: [2023-02-28 09:01:00,702] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +4: [2023-02-28 09:01:00,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:00,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +4: [2023-02-28 09:01:00,703] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +3: [2023-02-28 09:01:00,704] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +5: [2023-02-28 09:01:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +5: [2023-02-28 09:01:00,705] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +0: [2023-02-28 09:01:00,706] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +0: [2023-02-28 09:01:00,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,706] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +3: [2023-02-28 09:01:00,708] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +6: [2023-02-28 09:01:00,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +1: [2023-02-28 09:01:00,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,709] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +1: [2023-02-28 09:01:00,709] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +4: [2023-02-28 09:01:00,709] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +5: [2023-02-28 09:01:00,710] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +4: [2023-02-28 09:01:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,710] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +4: [2023-02-28 09:01:00,710] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +1: [2023-02-28 09:01:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-28 09:01:00,711] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +4: [2023-02-28 09:01:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-02-28 09:01:00,711] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +6: [2023-02-28 09:01:00,713] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +1: [2023-02-28 09:01:00,713] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +4: [2023-02-28 09:01:00,714] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +1: [2023-02-28 09:01:00,715] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +4: [2023-02-28 09:01:00,716] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +7: [2023-02-28 09:01:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,717] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +7: [2023-02-28 09:01:00,721] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +2: [2023-02-28 09:01:00,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,728] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +2: [2023-02-28 09:01:00,732] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +2: [2023-02-28 09:01:00,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,734] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +1: [2023-02-28 09:01:00,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,738] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +0: [2023-02-28 09:01:00,738] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +1: [2023-02-28 09:01:00,738] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +1: [2023-02-28 09:01:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-28 09:01:00,741] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +6: [2023-02-28 09:01:00,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:00,741] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +0: [2023-02-28 09:01:00,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,742] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +0: [2023-02-28 09:01:00,742] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +1: [2023-02-28 09:01:00,742] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +6: [2023-02-28 09:01:00,745] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +0: [2023-02-28 09:01:00,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,746] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +0: [2023-02-28 09:01:00,746] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +1: [2023-02-28 09:01:00,746] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +7: [2023-02-28 09:01:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,749] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +0: [2023-02-28 09:01:00,750] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +7: [2023-02-28 09:01:00,753] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +1: [2023-02-28 09:01:00,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-28 09:01:00,756] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +7: [2023-02-28 09:01:00,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,757] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +2: [2023-02-28 09:01:00,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,759] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +5: [2023-02-28 09:01:00,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +5: [2023-02-28 09:01:00,759] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +1: [2023-02-28 09:01:00,760] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +7: [2023-02-28 09:01:00,761] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +2: [2023-02-28 09:01:00,763] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +5: [2023-02-28 09:01:00,764] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +7: [2023-02-28 09:01:00,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,766] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +2: [2023-02-28 09:01:00,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,766] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +0: [2023-02-28 09:01:00,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,768] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +2: [2023-02-28 09:01:00,770] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +7: [2023-02-28 09:01:00,771] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +0: [2023-02-28 09:01:00,773] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +0: [2023-02-28 09:01:00,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-28 09:01:00,778] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +5: [2023-02-28 09:01:00,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-28 09:01:00,779] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +2: [2023-02-28 09:01:00,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,783] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +0: [2023-02-28 09:01:00,783] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +3: [2023-02-28 09:01:00,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:00,783] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +5: [2023-02-28 09:01:00,784] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +2: [2023-02-28 09:01:00,787] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +3: [2023-02-28 09:01:00,788] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +2: [2023-02-28 09:01:00,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-02-28 09:01:00,789] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +5: [2023-02-28 09:01:00,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-28 09:01:00,790] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +2: [2023-02-28 09:01:00,793] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +5: [2023-02-28 09:01:00,796] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +7: [2023-02-28 09:01:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-28 09:01:00,827] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +7: [2023-02-28 09:01:00,831] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +6: [2023-02-28 09:01:01,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-02-28 09:01:01,481] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +6: [2023-02-28 09:01:01,486] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +3: [2023-02-28 09:01:01,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b100m/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-28 09:01:01,497] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +3: [2023-02-28 09:01:01,501] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +0: successfully loaded checkpoint from checkpoints_619m22b100m at iteration 0 +7: time (ms) | load-checkpoint: 5753.25 +0: estimated model parameters: 0.618714624 +0: estimated model parameters without embeddings: 0.538301952 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-28 09:01:01 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.006928 seconds +0: number of documents: 208931 +0: > dataset split: +0: train: +0: document indices in [0, 208931) total of 208931 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_100M_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.006 seconds +0: total number of samples: 48805 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.064108 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.011 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-28 09:01:14 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 22192.68 | train/valid/test-data-iterators-setup: 12257.69 +0: [after training is done] datetime: 2023-02-28 09:01:14 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 6.840158E+00 | lm loss PPL: 9.346368E+02 | +7: ----------------------------------------------------------------------------------------------------------------- +END 3168317: Tue 28 Feb 2023 09:01:46 AM EET diff --git a/619m22b100m/sbatch_619m22b100m.sh b/619m22b100m/sbatch_619m22b100m.sh new file mode 100644 index 0000000000000000000000000000000000000000..bf69e9d3229eaf0702b77003c51fce92bcfd4d48 --- /dev/null +++ b/619m22b100m/sbatch_619m22b100m.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m22b100m + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train100m.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=5000 + +# Tokens: 21500000000 +# -> Samples: 10498047 +TRAIN_SAMPLES=10_498_047 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 104_980 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m22b100m/sbatch_619m22b100mval.sh b/619m22b100m/sbatch_619m22b100mval.sh new file mode 100644 index 0000000000000000000000000000000000000000..c9e5f587b436e7d16b6ac43f50e640025dc5ca8f --- /dev/null +++ b/619m22b100m/sbatch_619m22b100mval.sh @@ -0,0 +1,167 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m22b100mval +VARIANT_CKPT=619m22b100m + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train100m.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 21500000000 +# -> Samples: 10498047 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --override-lr-scheduler \ + --no-load-optim \ + --reset-progress \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m22b100m/tensorboard_619m22b100m/events.out.tfevents.1677516872.nid005441.11176.0 b/619m22b100m/tensorboard_619m22b100m/events.out.tfevents.1677516872.nid005441.11176.0 new file mode 100644 index 0000000000000000000000000000000000000000..46a2de7a619103ab317f3fd25d72c075dc3d43b2 --- /dev/null +++ b/619m22b100m/tensorboard_619m22b100m/events.out.tfevents.1677516872.nid005441.11176.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8989c85b0384557ac6961ea8c6b26c43a78f8ceecff71fb696d59eaf97fb81a6 +size 73304929 diff --git a/619m22b100m/tensorboard_619m22b100mval/events.out.tfevents.1677567618.nid005954.67979.0 b/619m22b100m/tensorboard_619m22b100mval/events.out.tfevents.1677567618.nid005954.67979.0 new file mode 100644 index 0000000000000000000000000000000000000000..772f2402f2b0c2c8dee50c6439465635e0c0a2ba --- /dev/null +++ b/619m22b100m/tensorboard_619m22b100mval/events.out.tfevents.1677567618.nid005954.67979.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:470be7519f8c07829fcb5247889d5c43b98e2db87f5f24b398724103f3e629a6 +size 980 diff --git a/619m22b1b5/logs/3162204.err b/619m22b1b5/logs/3162204.err new file mode 100644 index 0000000000000000000000000000000000000000..279f419a8e827320135ad65bacdfc35b476b9ed0 --- /dev/null +++ b/619m22b1b5/logs/3162204.err @@ -0,0 +1,1124 @@ +6: 2023-02-27 12:23:47.784014: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:47.784022: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:47.784013: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:47.784007: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:47.784021: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:47.784014: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:47.784020: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:47.784007: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: 2023-02-27 12:23:47.784318: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-27 12:23:47.784320: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-27 12:23:47.784317: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-27 12:23:47.784325: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-27 12:23:47.784328: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-27 12:23:47.784313: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-27 12:23:47.784333: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-27 12:23:47.796082: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802501: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802509: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802508: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802516: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802511: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802519: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802500: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-27 12:23:47.802516: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816282: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816283: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816276: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816275: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816291: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816287: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-27 12:23:47.816301: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864952: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864951: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864947: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864952: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864943: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864944: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864943: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-27 12:23:47.864945: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865416: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865419: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865423: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865429: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865431: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865427: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865416: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-27 12:23:47.865430: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924737: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924738: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924743: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924748: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924743: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924744: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-27 12:23:47.924755: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016848: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016857: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016846: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016853: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016856: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016860: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016853: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-27 12:23:48.016858: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-27 12:23:49.403677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.403690: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.403690: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.403697: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.403699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.403694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.403690: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.403696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:49.404048: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-27 12:23:49.404051: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-27 12:23:49.404054: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-27 12:23:49.404057: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-27 12:23:49.404060: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-27 12:23:49.404060: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-27 12:23:49.404063: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-27 12:23:49.404067: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404513: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404515: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:49.404910: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404912: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404917: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404919: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404921: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404921: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404924: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-27 12:23:49.404933: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435258: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:49.435655: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435661: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435662: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435663: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435668: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435668: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-27 12:23:49.435671: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440022: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:49.440466: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440469: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440472: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440475: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440476: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440476: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440480: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-27 12:23:49.440483: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457080: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457080: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457083: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:49.457290: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457290: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457296: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457297: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457297: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457302: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457303: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-27 12:23:49.457304: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526299: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526309: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526313: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526313: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:49.526659: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526661: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526666: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526667: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526668: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526670: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526671: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:49.526679: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.684674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.684676: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.684688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.684686: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.684691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.684698: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.684699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.684701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:49.685117: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.685124: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.685126: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.685127: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.685130: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.685130: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.685131: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-27 12:23:49.685134: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726016: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726021: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726021: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:49.726435: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726442: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726444: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726445: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726449: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726450: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726451: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-27 12:23:49.726452: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-27 12:23:54.465652: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.465648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.465659: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.465658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.465663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.465661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.465666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.465667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.466052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-27 12:23:54.466105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.466060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.466109: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-27 12:23:54.466066: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.466114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-27 12:23:54.466063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.466117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-27 12:23:54.466061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.466119: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-27 12:23:54.466059: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.466119: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-27 12:23:54.466068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.466125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-27 12:23:54.466072: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.466172: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-27 12:23:54.466122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.466175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.466180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.466183: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.466456: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: 2023-02-27 12:23:54.466182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.466189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: 2023-02-27 12:23:54.466465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.466188: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: 2023-02-27 12:23:54.466464: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.466194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: 2023-02-27 12:23:54.466469: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.466473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466611: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.466474: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.466473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-27 12:23:54.466664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466617: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.466473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.466660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466624: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.466667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466625: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.466671: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.466673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: 2023-02-27 12:23:54.466627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.466671: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.466679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.466676: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467640: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467641: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467639: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467641: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-27 12:23:54.467657: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-27 12:23:54.467659: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-27 12:23:54.467660: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-27 12:23:54.467661: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-27 12:23:54.467662: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-27 12:23:54.467664: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-27 12:23:54.467665: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-27 12:23:54.467666: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468022: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468024: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468037: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468039: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468040: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468043: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468044: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468046: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468047: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-27 12:23:54.468080: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-27 12:23:54.468094: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-27 12:23:54.468390: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-27 12:23:54.468480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.468546: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-27 12:23:54.468515: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-27 12:23:54.468484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468395: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:54.468485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:54.468494: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-27 12:23:54.468395: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: 2023-02-27 12:23:54.468551: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-27 12:23:54.468516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-27 12:23:54.468488: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468406: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-27 12:23:54.468407: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468409: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-27 12:23:54.468411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-27 12:23:54.468411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-27 12:23:54.468555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-27 12:23:54.468516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-27 12:23:54.468490: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-27 12:23:54.468413: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-27 12:23:54.468415: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468427: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: 2023-02-27 12:23:54.468555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-27 12:23:54.468521: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-27 12:23:54.468489: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-27 12:23:54.468439: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.468558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-27 12:23:54.468520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-27 12:23:54.468489: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:54.468498: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-27 12:23:54.468499: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-27 12:23:54.468496: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-27 12:23:54.468508: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-27 12:23:54.468509: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.468518: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: 2023-02-27 12:23:54.468509: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-27 12:23:54.468559: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.468530: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-27 12:23:54.468511: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-27 12:23:54.468513: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.468564: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-27 12:23:54.468532: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-27 12:23:54.468535: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-27 12:23:54.468537: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-27 12:23:54.468562: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-27 12:23:54.468537: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-27 12:23:54.468538: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-27 12:23:54.468563: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: 2023-02-27 12:23:54.468565: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.468571: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-27 12:23:54.468574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-27 12:23:54.468565: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: 2023-02-27 12:23:54.468574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-27 12:23:54.468577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-27 12:23:54.468577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-27 12:23:54.468579: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-27 12:23:54.468581: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-27 12:23:54.468577: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-27 12:23:54.468578: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.481635: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.481648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.481645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.481643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.481651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.481651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.481654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.481658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483187: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483201: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.483201: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.483197: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483197: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483202: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483213: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.483218: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.483213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-27 12:23:54.483220: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.483221: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.483222: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-27 12:23:54.483229: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468600: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468602: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468604: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468605: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468611: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468612: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468616: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468618: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468618: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468618: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468624: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-27 12:23:54.468627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-27 12:23:54.468639: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_upper_triang_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +0: Successfully preprocessed all matching files. +6: Successfully preprocessed all matching files. +6: Successfully preprocessed all matching files. +6: Successfully preprocessed all matching files. +7: Successfully preprocessed all matching files. +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +0: Building extension module utils... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: +1: +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: +5: +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +1: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +1: Building extension module utils... +1: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +1: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +6: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +3: Loading extension module utils... +6: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +6: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +4: Loading extension module utils... +3: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +0: +0: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +2: Loading extension module utils... +7: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +2: +2: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils... +2: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +4: Loading extension module utils... +4: +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +3: +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Loading extension module utils... +7: +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: Loading extension module utils... +7: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +6: +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +6: +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/619m22b1b5/logs/3162204.out b/619m22b1b5/logs/3162204.out new file mode 100644 index 0000000000000000000000000000000000000000..65e513db3051a95ca97dab2f2a0eb2376eb31c47 --- /dev/null +++ b/619m22b1b5/logs/3162204.out @@ -0,0 +1,6693 @@ +Model parameters: d_model 1536 ffw_size 6144 kv_size 128 n_heads 12 n_layers 19 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 19 --hidden-size 1536 --num-attention-heads 12 --kv-channels 128 --ffn-hidden-size 6144 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-619m22b1b5val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --no-load-optim --reset-progress --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --tensorboard-dir tensorboard_619m22b1b5val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_619m22b1b5 --load checkpoints_619m22b1b5 --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3162204.json --zero-stage 0 +START 3162204: Mon 27 Feb 2023 12:23:30 PM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 43.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 48.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 48.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 53.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 41.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 40.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 39.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 47.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 40.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 43.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 41.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +3: Launching on nid005793 (3/8), master nid005790 port 9999, GPUs 8, CUDA: True +1: Launching on nid005791 (1/8), master nid005790 port 9999, GPUs 8, CUDA: True +4: Launching on nid005794 (4/8), master nid005790 port 9999, GPUs 8, CUDA: True +2: Launching on nid005792 (2/8), master nid005790 port 9999, GPUs 8, CUDA: True +0: Launching on nid005790 (0/8), master nid005790 port 9999, GPUs 8, CUDA: True +5: Launching on nid005795 (5/8), master nid005790 port 9999, GPUs 8, CUDA: True +6: Launching on nid005796 (6/8), master nid005790 port 9999, GPUs 8, CUDA: True +7: Launching on nid005797 (7/8), master nid005790 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/3162204.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... None +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 6144 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1536 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-619m22b1b5val +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_619m22b1b5 +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... None +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 12 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 19 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_619m22b1b5 +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_619m22b1b5val +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +7: > setting tensorboard ... +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-27 12:24:19,313] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.111 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 87 +0: ninja: no work to do. +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 63 +0: ninja: no work to do. +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: ninja: no work to do. +0: >>> done with compiling and loading fused kernels. Compilation time: 17.992 seconds +0: time to initialize megatron (seconds): -23.892 +0: [after megatron is initialized] datetime: 2023-02-27 12:24:40 +0: building GPT model ... +0: [2023-02-27 12:24:40,248] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-27 12:24:40,248] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-27 12:24:40,249] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 33.33 GB, percent = 6.6% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-27 12:24:42,220] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=26 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: undo +0: 23: MixedFusedLayerNorm +0: 24: EmbeddingPipe +0: 25: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-27 12:24:42,432] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-27 12:24:42,433] [INFO] [utils.py:828:see_memory_usage] MA 1.16 GB Max_MA 1.16 GB CA 1.2 GB Max_CA 1 GB +0: [2023-02-27 12:24:42,433] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 33.37 GB, percent = 6.6% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-27 12:24:42,435] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-27 12:24:55,225] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-27 12:24:55,225] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-27 12:24:55,225] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-27 12:24:55,232] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-27 12:24:55,232] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-27 12:24:55,353] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-27 12:24:55,353] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.17 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-27 12:24:55,353] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.06 GB, percent = 6.8% +0: ninja: no work to do. +0: Time to load utils op: 0.272235631942749 seconds +0: Time to load utils op: 0.000986337661743164 seconds +1: ninja: no work to do. +1: Time to load utils op: 0.2823178768157959 seconds +0: Time to load utils op: 0.30230212211608887 seconds +0: Time to load utils op: 0.30272841453552246 seconds +0: Time to load utils op: 0.3026144504547119 seconds +0: Time to load utils op: 0.3024778366088867 seconds +0: Time to load utils op: 0.3033773899078369 seconds +0: Time to load utils op: 0.30327892303466797 seconds +1: Time to load utils op: 0.3041210174560547 seconds +1: Time to load utils op: 0.3051433563232422 secondsTime to load utils op: 0.30512142181396484 seconds +1: +1: Time to load utils op: 0.30518436431884766 seconds +1: Time to load utils op: 0.3051717281341553 seconds +1: Time to load utils op: 0.30532002449035645 seconds +1: Time to load utils op: 0.3054311275482178 seconds +2: Time to load utils op: 0.30867886543273926 seconds +2: Time to load utils op: 0.3086392879486084 seconds +2: Time to load utils op: 0.30866098403930664 seconds +2: Time to load utils op: 0.3086733818054199 seconds +2: Time to load utils op: 0.3087160587310791 seconds +2: Time to load utils op: 0.309650182723999 seconds +2: Time to load utils op: 0.3091297149658203 seconds +0: Time to load utils op: 0.0005161762237548828 seconds +6: Time to load utils op: 0.3133711814880371 seconds +6: Time to load utils op: 0.312824010848999 seconds +6: Time to load utils op: 0.312908411026001 seconds +6: Time to load utils op: 0.3122420310974121 seconds +6: Time to load utils op: 0.3131735324859619 seconds +6: Time to load utils op: 0.31299424171447754 secondsTime to load utils op: 0.3122556209564209 seconds +6: +6: Time to load utils op: 0.3131124973297119 seconds +0: Time to load utils op: 0.0003669261932373047 seconds +3: Time to load utils op: 0.3122575283050537 secondsTime to load utils op: 0.3122901916503906 seconds +3: Time to load utils op: 0.3123025894165039 seconds +3: +3: Time to load utils op: 0.31230974197387695 seconds +3: Time to load utils op: 0.31228184700012207 secondsTime to load utils op: 0.3123049736022949 seconds +3: +3: Time to load utils op: 0.3123204708099365 seconds +3: Time to load utils op: 0.31229567527770996 seconds +0: Time to load utils op: 0.00036597251892089844 seconds +0: Time to load utils op: 0.00040149688720703125 seconds +0: Time to load utils op: 0.0004031658172607422 seconds +7: Time to load utils op: 0.31067609786987305 seconds +7: Time to load utils op: 0.3107020854949951 seconds +7: Time to load utils op: 0.3104987144470215 seconds +7: Time to load utils op: 0.30983734130859375 secondsTime to load utils op: 0.3109397888183594 seconds +7: +7: Time to load utils op: 0.3097116947174072 secondsTime to load utils op: 0.3108677864074707 seconds +7: +4: Time to load utils op: 0.3128325939178467 seconds +4: Time to load utils op: 0.3128395080566406 seconds +0: Time to load utils op: 0.00039196014404296875 seconds +4: Time to load utils op: 0.31284213066101074 seconds +4: Time to load utils op: 0.31285643577575684 seconds +4: Time to load utils op: 0.3128817081451416 seconds +4: Time to load utils op: 0.31286096572875977 seconds +4: Time to load utils op: 0.3128950595855713 secondsTime to load utils op: 0.3129146099090576 seconds +4: +5: Time to load utils op: 0.31110358238220215 secondsTime to load utils op: 0.3111133575439453 seconds +5: +5: Time to load utils op: 0.31113672256469727 secondsTime to load utils op: 0.31112051010131836 seconds +5: +5: Time to load utils op: 0.3111283779144287 seconds +5: Time to load utils op: 0.31114864349365234 seconds +5: Time to load utils op: 0.31116366386413574 seconds +5: Time to load utils op: 0.3111867904663086 seconds +2: Time to load utils op: 0.6044597625732422 seconds +7: Time to load utils op: 0.604283332824707 seconds +1: Time to load utils op: 0.0007295608520507812 seconds +1: Time to load utils op: 0.0007119178771972656 seconds +1: Time to load utils op: 0.0010592937469482422 secondsTime to load utils op: 0.001081705093383789 seconds +1: +1: Time to load utils op: 0.0009922981262207031 seconds +1: Time to load utils op: 0.0009927749633789062 seconds +1: Time to load utils op: 0.0010862350463867188 seconds +1: Time to load utils op: 0.0008113384246826172 seconds +0: Time to load utils op: 0.5034232139587402 seconds +5: Time to load utils op: 0.0009295940399169922 seconds +2: Time to load utils op: 0.0005106925964355469 seconds +2: Time to load utils op: 0.0005438327789306641 seconds +2: Time to load utils op: 0.0005404949188232422 secondsTime to load utils op: 0.0005548000335693359 seconds +2: +2: Time to load utils op: 0.0005028247833251953 seconds +2: Time to load utils op: 0.0005369186401367188 seconds +5: Time to load utils op: 0.0013110637664794922 seconds +5: Time to load utils op: 0.0012936592102050781 seconds +5: Time to load utils op: 0.001298666000366211 seconds +5: Time to load utils op: 0.0013055801391601562 seconds +5: Time to load utils op: 0.0013897418975830078 secondsTime to load utils op: 0.0013127326965332031 seconds +5: +5: Time to load utils op: 0.0013432502746582031 seconds +4: Time to load utils op: 0.0008232593536376953 seconds +4: Time to load utils op: 0.0007798671722412109 seconds +4: Time to load utils op: 0.0010180473327636719 seconds +4: Time to load utils op: 0.0008330345153808594 seconds +4: Time to load utils op: 0.0007143020629882812 secondsTime to load utils op: 0.0007033348083496094 secondsTime to load utils op: 0.000698089599609375 seconds +4: +4: +4: Time to load utils op: 0.0006766319274902344 seconds +3: Time to load utils op: 0.0008473396301269531 seconds +3: Time to load utils op: 0.0012221336364746094 seconds +3: Time to load utils op: 0.0011680126190185547 secondsTime to load utils op: 0.0011844635009765625 seconds +3: +3: Time to load utils op: 0.0011510848999023438 seconds +3: Time to load utils op: 0.0011730194091796875 seconds +3: Time to load utils op: 0.0012111663818359375 seconds +3: Time to load utils op: 0.0012385845184326172 seconds +7: Time to load utils op: 0.000560760498046875 seconds +7: Time to load utils op: 0.0003986358642578125 seconds +7: Time to load utils op: 0.00041794776916503906 seconds +7: Time to load utils op: 0.0005121231079101562 seconds +7: Time to load utils op: 0.0004203319549560547 seconds +7: Time to load utils op: 0.0004172325134277344 seconds +7: Time to load utils op: 0.0004029273986816406 secondsTime to load utils op: 0.00042247772216796875 seconds +7: +6: Time to load utils op: 0.0007965564727783203 seconds +6: Time to load utils op: 0.001018524169921875 seconds +6: Time to load utils op: 0.0011947154998779297 seconds +6: Time to load utils op: 0.0011212825775146484 seconds +6: Time to load utils op: 0.001367330551147461 seconds +6: Time to load utils op: 0.0013704299926757812 secondsTime to load utils op: 0.0013041496276855469 seconds +6: +6: Time to load utils op: 0.0014410018920898438 seconds +2: Time to load utils op: 0.0006012916564941406 seconds +2: Time to load utils op: 0.0005872249603271484 seconds +0: [2023-02-27 12:24:55,972] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-02-27 12:24:55,973] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.15 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-27 12:24:55,973] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,085] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-27 12:24:56,086] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-27 12:24:56,086] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,187] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-27 12:24:56,188] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-27 12:24:56,188] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,290] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-27 12:24:56,291] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-27 12:24:56,291] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,392] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-27 12:24:56,392] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-27 12:24:56,392] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,495] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-27 12:24:56,495] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-27 12:24:56,495] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,595] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-27 12:24:56,595] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-27 12:24:56,596] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,701] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-27 12:24:56,702] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-27 12:24:56,702] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,802] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-27 12:24:56,803] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-27 12:24:56,803] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 34.2 GB, percent = 6.8% +0: [2023-02-27 12:24:56,803] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-27 12:24:56,803] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-27 12:24:56,803] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-27 12:24:56,803] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-27 12:24:56,803] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-27 12:24:56,804] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-27 12:24:56,805] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-27 12:24:56,806] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-27 12:24:56,806] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-27 12:24:56,806] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-27 12:24:56,806] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.00042319297790527344 seconds +0: [2023-02-27 12:24:56,806] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-27 12:24:56,881] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=26 [0, 26) STAGE_PARAMS=618714624 (618.715M) TOTAL_PARAMS=618714624 (618.715M) UNIQUE_PARAMS=618714624 (618.715M) +0: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-27 12:24:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:57,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-27 12:24:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-27 12:24:57,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-27 12:24:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-27 12:24:57,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-27 12:24:57,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:57,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:57,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:57,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-27 12:24:57,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-27 12:24:57,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-27 12:24:57,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-27 12:24:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-27 12:24:57,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-27 12:24:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-27 12:24:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-27 12:24:57,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-27 12:24:57,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-27 12:24:57,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-27 12:24:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-27 12:24:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-27 12:24:57,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-27 12:24:57,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-27 12:24:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-27 12:24:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-27 12:24:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-27 12:24:57,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-27 12:24:57,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-27 12:24:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-27 12:24:57,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-27 12:24:57,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-27 12:24:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-27 12:24:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-27 12:24:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-27 12:24:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-27 12:24:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-27 12:24:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-27 12:24:57,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-27 12:24:57,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-27 12:24:57,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-27 12:24:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-27 12:24:57,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-27 12:24:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-27 12:24:57,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-27 12:24:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:57,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-27 12:24:57,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-27 12:24:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-27 12:24:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:57,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:57,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-27 12:24:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-27 12:24:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-27 12:24:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-27 12:24:57,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-27 12:24:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-27 12:24:57,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:57,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-27 12:24:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:58,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:58,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:58,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:58,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-27 12:24:58,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:58,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-27 12:24:58,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-27 12:24:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-27 12:24:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-27 12:24:58,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-27 12:24:58,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-27 12:24:58,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-27 12:24:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-27 12:24:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-27 12:24:58,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-27 12:24:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-27 12:24:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-27 12:24:58,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-27 12:24:58,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-27 12:24:58,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-27 12:24:58,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-27 12:24:58,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-27 12:24:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-27 12:24:58,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-27 12:24:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-27 12:24:58,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-27 12:24:58,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-27 12:24:58,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-27 12:24:58,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-27 12:24:58,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-27 12:24:58,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-27 12:24:58,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-27 12:24:58,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-27 12:24:58,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-27 12:24:58,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-27 12:24:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-27 12:24:58,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-27 12:24:58,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-27 12:24:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-27 12:24:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-27 12:24:58,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-27 12:24:58,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-27 12:24:58,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-27 12:24:58,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-27 12:24:58,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-27 12:24:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-27 12:24:58,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-27 12:24:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-27 12:24:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-27 12:24:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-27 12:24:58,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-27 12:24:58,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-27 12:24:58,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-27 12:24:58,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-27 12:24:58,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-27 12:24:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-27 12:24:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-27 12:24:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-27 12:24:58,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-27 12:24:58,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-27 12:24:58,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-27 12:24:58,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-27 12:24:58,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-27 12:24:58,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-27 12:24:58,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-27 12:24:58,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-27 12:24:58,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-27 12:24:58,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-27 12:24:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-27 12:24:58,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-27 12:24:58,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-27 12:24:58,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-27 12:24:58,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-27 12:24:58,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-27 12:24:58,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-27 12:24:58,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-27 12:24:58,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-27 12:24:58,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-27 12:24:58,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-27 12:24:58,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-27 12:24:58,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-27 12:24:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-27 12:24:58,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-27 12:24:58,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-27 12:24:58,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-27 12:24:58,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-27 12:24:58,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-27 12:24:58,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-27 12:24:58,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-27 12:24:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-27 12:24:58,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-27 12:24:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-27 12:24:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-27 12:24:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-27 12:24:59,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:59,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-27 12:24:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:59,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:59,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:59,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:59,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:59,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-27 12:24:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-27 12:24:59,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:59,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:59,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:59,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:59,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-27 12:24:59,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-27 12:24:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-27 12:24:59,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-27 12:24:59,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-27 12:24:59,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-27 12:24:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-27 12:24:59,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-27 12:24:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-27 12:24:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-27 12:24:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-27 12:24:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-27 12:24:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-27 12:24:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-27 12:24:59,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-27 12:24:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-27 12:24:59,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-27 12:24:59,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-27 12:24:59,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-27 12:24:59,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-27 12:24:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-27 12:24:59,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-27 12:24:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-27 12:24:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-27 12:24:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-27 12:24:59,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-27 12:24:59,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-27 12:24:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-27 12:24:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-27 12:24:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-27 12:24:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-27 12:24:59,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-27 12:24:59,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-27 12:24:59,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-27 12:24:59,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-27 12:24:59,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-27 12:24:59,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-27 12:24:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-27 12:24:59,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-27 12:24:59,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-27 12:24:59,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-27 12:24:59,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-27 12:24:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-27 12:24:59,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-27 12:24:59,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-27 12:24:59,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-27 12:24:59,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-27 12:24:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-27 12:24:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-27 12:24:59,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-27 12:24:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-27 12:24:59,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-27 12:24:59,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-27 12:24:59,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-27 12:24:59,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-27 12:24:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-27 12:24:59,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-27 12:24:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-27 12:24:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-27 12:24:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-27 12:24:59,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-27 12:24:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:24:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:24:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-27 12:24:59,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-27 12:24:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-27 12:24:59,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-27 12:24:59,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-27 12:24:59,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-27 12:24:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-27 12:24:59,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-27 12:24:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:24:59,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:24:59,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:24:59,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:24:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-27 12:24:59,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-27 12:24:59,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:24:59,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:24:59,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:24:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-27 12:24:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-27 12:24:59,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:24:59,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:24:59,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-27 12:24:59,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-27 12:24:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:24:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:24:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:24:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-27 12:24:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:25:00,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:25:00,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-27 12:25:00,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-27 12:25:00,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-27 12:25:00,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-27 12:25:00,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-27 12:25:00,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:25:00,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:25:00,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:25:00,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-27 12:25:00,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-27 12:25:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-27 12:25:00,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-27 12:25:00,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-27 12:25:00,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-27 12:25:00,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-27 12:25:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-27 12:25:00,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-27 12:25:00,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-27 12:25:00,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-27 12:25:00,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-27 12:25:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-27 12:25:00,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-27 12:25:00,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-27 12:25:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-27 12:25:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-27 12:25:00,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-27 12:25:00,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-27 12:25:00,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-27 12:25:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-27 12:25:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-27 12:25:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-27 12:25:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-27 12:25:00,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-27 12:25:00,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-27 12:25:00,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-27 12:25:00,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-27 12:25:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-27 12:25:00,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-27 12:25:00,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-27 12:25:00,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-27 12:25:00,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-27 12:25:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-27 12:25:00,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-27 12:25:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-27 12:25:00,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-27 12:25:00,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-27 12:25:00,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-27 12:25:00,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-27 12:25:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-27 12:25:00,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-27 12:25:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-27 12:25:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-27 12:25:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-02-27 12:25:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +2: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +2: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +2: [2023-02-27 12:25:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +3: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +1: [2023-02-27 12:25:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-27 12:25:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +4: [2023-02-27 12:25:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +6: [2023-02-27 12:25:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +2: [2023-02-27 12:25:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-27 12:25:00,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-27 12:25:00,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-27 12:25:00,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-27 12:25:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +0: [2023-02-27 12:25:00,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-27 12:25:00,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-27 12:25:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-27 12:25:00,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-27 12:25:00,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +4: [2023-02-27 12:25:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +7: [2023-02-27 12:25:00,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:00,983] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +7: [2023-02-27 12:25:00,987] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +5: [2023-02-27 12:25:01,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-02-27 12:25:01,025] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +5: [2023-02-27 12:25:01,028] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +0: [2023-02-27 12:25:01,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,037] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +1: [2023-02-27 12:25:01,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,042] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +0: [2023-02-27 12:25:01,042] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +7: [2023-02-27 12:25:01,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:01,044] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +1: [2023-02-27 12:25:01,046] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +7: [2023-02-27 12:25:01,048] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +6: [2023-02-27 12:25:01,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-27 12:25:01,057] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +6: [2023-02-27 12:25:01,061] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +5: [2023-02-27 12:25:01,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +5: [2023-02-27 12:25:01,069] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +3: [2023-02-27 12:25:01,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,070] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +2: [2023-02-27 12:25:01,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,072] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +3: [2023-02-27 12:25:01,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,073] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +5: [2023-02-27 12:25:01,074] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +3: [2023-02-27 12:25:01,074] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +4: [2023-02-27 12:25:01,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,074] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +2: [2023-02-27 12:25:01,076] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +3: [2023-02-27 12:25:01,077] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +4: [2023-02-27 12:25:01,078] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +1: [2023-02-27 12:25:01,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,079] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +1: [2023-02-27 12:25:01,084] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +1: [2023-02-27 12:25:01,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,088] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +1: [2023-02-27 12:25:01,092] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +0: [2023-02-27 12:25:01,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,098] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +5: [2023-02-27 12:25:01,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-27 12:25:01,099] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +0: [2023-02-27 12:25:01,101] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +5: [2023-02-27 12:25:01,103] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +3: [2023-02-27 12:25:01,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,104] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +0: [2023-02-27 12:25:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,107] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +6: [2023-02-27 12:25:01,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-27 12:25:01,108] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +7: [2023-02-27 12:25:01,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:01,108] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +5: [2023-02-27 12:25:01,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-27 12:25:01,108] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +3: [2023-02-27 12:25:01,109] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +6: [2023-02-27 12:25:01,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +0: [2023-02-27 12:25:01,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +7: [2023-02-27 12:25:01,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +5: [2023-02-27 12:25:01,113] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +2: [2023-02-27 12:25:01,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,116] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +1: [2023-02-27 12:25:01,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,118] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +2: [2023-02-27 12:25:01,120] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +1: [2023-02-27 12:25:01,122] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +4: [2023-02-27 12:25:01,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,125] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +1: [2023-02-27 12:25:01,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,128] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +4: [2023-02-27 12:25:01,129] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +1: [2023-02-27 12:25:01,132] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +0: [2023-02-27 12:25:01,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +5: [2023-02-27 12:25:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,142] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +5: [2023-02-27 12:25:01,142] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +4: [2023-02-27 12:25:01,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,142] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +3: [2023-02-27 12:25:01,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,143] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +7: [2023-02-27 12:25:01,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:01,145] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +5: [2023-02-27 12:25:01,146] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +3: [2023-02-27 12:25:01,146] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +4: [2023-02-27 12:25:01,147] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +4: [2023-02-27 12:25:01,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:01,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,148] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +7: [2023-02-27 12:25:01,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +7: [2023-02-27 12:25:01,149] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +7: [2023-02-27 12:25:01,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:01,150] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +7: [2023-02-27 12:25:01,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:01,150] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +2: [2023-02-27 12:25:01,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,152] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +4: [2023-02-27 12:25:01,153] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +7: [2023-02-27 12:25:01,153] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +6: [2023-02-27 12:25:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-02-27 12:25:01,154] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +7: [2023-02-27 12:25:01,154] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +7: [2023-02-27 12:25:01,155] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +2: [2023-02-27 12:25:01,156] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +6: [2023-02-27 12:25:01,158] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +4: [2023-02-27 12:25:01,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,160] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +6: [2023-02-27 12:25:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +6: [2023-02-27 12:25:01,161] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +0: [2023-02-27 12:25:01,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,162] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +6: [2023-02-27 12:25:01,165] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +3: [2023-02-27 12:25:01,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,166] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +4: [2023-02-27 12:25:01,166] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +0: [2023-02-27 12:25:01,167] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +4: [2023-02-27 12:25:01,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,168] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +1: [2023-02-27 12:25:01,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,170] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +3: [2023-02-27 12:25:01,170] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +4: [2023-02-27 12:25:01,172] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +6: [2023-02-27 12:25:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-02-27 12:25:01,172] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +2: [2023-02-27 12:25:01,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,173] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +4: [2023-02-27 12:25:01,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,174] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +1: [2023-02-27 12:25:01,174] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +3: [2023-02-27 12:25:01,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,175] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +2: [2023-02-27 12:25:01,177] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +4: [2023-02-27 12:25:01,178] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +3: [2023-02-27 12:25:01,179] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +5: [2023-02-27 12:25:01,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-27 12:25:01,180] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +6: [2023-02-27 12:25:01,177] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +5: [2023-02-27 12:25:01,184] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +2: [2023-02-27 12:25:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,186] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +0: [2023-02-27 12:25:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,187] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +0: [2023-02-27 12:25:01,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,188] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +2: [2023-02-27 12:25:01,190] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +0: [2023-02-27 12:25:01,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,191] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +2: [2023-02-27 12:25:01,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,193] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +2: [2023-02-27 12:25:01,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,193] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +0: [2023-02-27 12:25:01,195] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +6: [2023-02-27 12:25:01,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +0: [2023-02-27 12:25:01,195] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +6: [2023-02-27 12:25:01,195] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +6: [2023-02-27 12:25:01,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-27 12:25:01,196] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +0: [2023-02-27 12:25:01,196] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +2: [2023-02-27 12:25:01,198] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +2: [2023-02-27 12:25:01,198] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +3: [2023-02-27 12:25:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +3: [2023-02-27 12:25:01,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +3: [2023-02-27 12:25:01,201] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +2: [2023-02-27 12:25:01,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-02-27 12:25:01,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +6: [2023-02-27 12:25:01,202] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +6: [2023-02-27 12:25:01,202] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +6: [2023-02-27 12:25:01,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-27 12:25:01,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +3: [2023-02-27 12:25:01,204] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +3: [2023-02-27 12:25:01,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +6: [2023-02-27 12:25:01,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +2: [2023-02-27 12:25:01,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +7: [2023-02-27 12:25:01,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-27 12:25:01,213] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +4: [2023-02-27 12:25:01,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-27 12:25:01,215] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +7: [2023-02-27 12:25:01,218] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +4: [2023-02-27 12:25:01,220] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +1: [2023-02-27 12:25:01,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +1: [2023-02-27 12:25:01,239] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +1: [2023-02-27 12:25:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-27 12:25:01,239] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +1: [2023-02-27 12:25:01,244] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +5: [2023-02-27 12:25:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +5: [2023-02-27 12:25:01,253] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +5: [2023-02-27 12:25:01,258] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +5: [2023-02-27 12:25:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b1b5/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-27 12:25:01,260] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +5: [2023-02-27 12:25:01,265] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +0: successfully loaded checkpoint from checkpoints_619m22b1b5 at iteration 0 +7: time (ms) | load-checkpoint: 4389.68 +0: estimated model parameters: 0.618714624 +0: estimated model parameters without embeddings: 0.538301952 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-27 12:25:01 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.006610 seconds +0: number of documents: 3133972 +0: > dataset split: +0: train: +0: document indices in [0, 3133972) total of 3133972 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.088 seconds +0: total number of samples: 731002 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.044450 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.076 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-27 12:25:15 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 21328.54 | train/valid/test-data-iterators-setup: 13611.26 +0: [after training is done] datetime: 2023-02-27 12:25:15 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 2.973545E+00 | lm loss PPL: 1.956114E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 3162204: Mon 27 Feb 2023 12:25:45 PM EET diff --git a/619m22b1b5/tensorboard_619m22b1b5/events.out.tfevents.1677456683.nid005981.58395.0 b/619m22b1b5/tensorboard_619m22b1b5/events.out.tfevents.1677456683.nid005981.58395.0 new file mode 100644 index 0000000000000000000000000000000000000000..9f46e611a5705ec839699c12dc8821f11261bdba --- /dev/null +++ b/619m22b1b5/tensorboard_619m22b1b5/events.out.tfevents.1677456683.nid005981.58395.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d898326916e54e2128b1057d4bd8421931eb32874ecec4b8d357bcdd33698a75 +size 73304924 diff --git a/619m22b1b5/tensorboard_619m22b1b5val/events.out.tfevents.1677493459.nid005797.98044.0 b/619m22b1b5/tensorboard_619m22b1b5val/events.out.tfevents.1677493459.nid005797.98044.0 new file mode 100644 index 0000000000000000000000000000000000000000..6c1168b0635d2d4258c3f7a6a14403bcd5620d26 --- /dev/null +++ b/619m22b1b5/tensorboard_619m22b1b5val/events.out.tfevents.1677493459.nid005797.98044.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b3f62b8c0afcf6b2759a94f93ef160b4288535572c26f0d195bfb386d020df9 +size 980 diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3c49a7e7fe1c868ee010c92b968a2d18f9e1f061 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.284489408206782, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.028278761736760676}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06358663835483319, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014889693861394535}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.27174214833640686, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00457462478813889}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09675681057276961, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001919696307619028}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.030422997079269397, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009323098094739095}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13467661456157262, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003054061694040572}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.046357243917834384, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012013193200828695}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.061672065465079716, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014080585643345802}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2661377360349057, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004497165496966715}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09413955269824087, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018351981952839998}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06091800415353282, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014171331193967864}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.260194740499195, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004313457116793654}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09267090154658023, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018160333067075032}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bc2c274dc3f021d252c5d9074c2f15232d36655c --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.30794107382055563, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.025784978769138028}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06665371404806113, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015734440957952153}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.3144424936447866, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0051206124313989}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.10120740105666726, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019092721691292443}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.030646030870042525, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001004143407358295}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.14761105493339954, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0032987706705033693}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04645784299011615, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011967388539773235}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06342458988254415, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001438569090801163}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.3009513725545963, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00484963269043915}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.0965472169328251, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017612040569045689}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06335872417977906, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001469450758465004}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2974222169844293, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004695371476880067}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09616343757930998, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017828648565307438}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b60c81f6381d33d017f1a5e2ecfd2e2bc9f0591d --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.2637530179976743, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.016909949021643642}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.0615020828840054, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013433273947903563}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.299445798213358, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004915871939991809}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09501611598218968, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001741927321610452}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02801441282828333, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008028230234396205}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.14194025928903903, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003182936083641798}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04336836166197178, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010640454351919701}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05880744102356161, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012482717064613974}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.28566216477217093, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004570189393285611}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09086602850993777, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016091969110186804}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.059093493415271564, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001289531903153165}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2860223803179543, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004582771242586325}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09117136970494266, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001656189124088108}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f63a0f1b0ab2346edec94b71de6e4182eaf12920 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.30127710888872544, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03017358616348081}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06104007013714008, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013491551372300523}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.29887189317521123, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004914239206319181}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.0945906963669224, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017693406808415686}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.027910230506178427, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000841081292987736}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1423375012631199, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0032267429123653078}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04336590682538455, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011190741529048644}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.058159150618774094, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001205587984308903}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2849440964479655, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004563998064685871}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09028317266790117, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016079490173524383}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05857104196241217, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012455598399315733}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.28603952371373775, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004599540876569589}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09077654667629809, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001652139375900535}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c447ee4da9c77710e25e6f0a299b49733c6d4aa4 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.2872597163393854, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.028404283672526617}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06131644179463926, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012834363684846703}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.3028350802402193, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0047300632430552995}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09539577595993691, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016905421986277095}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.028022710412489778, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007615815376667828}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.14606820900953005, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003196813879267001}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04381547514401627, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010528299563527775}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05845302564855468, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011796896274871784}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2878847835539455, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004378448457675036}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09097440961486976, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015665857329272954}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.058888194545668245, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012066969569740117}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.29032520701751985, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004449008541788328}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09161480579427395, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015993125551969124}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..73100d9d3ca1cd897a986effa6e7403c8d5fd21a --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.31000340585989805, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.035124165599714846}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06197295473456213, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012498429939017263}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.30771780339869886, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0047481327408507015}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09691863328157169, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016948022167593932}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02844038599993891, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007609587837791093}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.15003058591952856, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003250392179692442}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04471206930702434, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010614115742719097}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.058896059472105716, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001142867644266226}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2918380879479885, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0043711606101936385}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09219347050075496, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015673266760248422}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05964055672474598, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011914634127819287}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2955051016060543, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004487785833382257}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09326972432857124, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016226044008928713}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..bb2cff5e7ccc32c5f6b27d5b8919c5db8cfd3ff3 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.13630335885749084, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018434565261148661}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.23093456995895148, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026298312792475474}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.15907081996190248, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018212459268317244}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.025089156441511235, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006575369718712219}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.045617278146314374, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013397315108689542}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.029634443330971802, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000753347585845674}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.10924764012514825, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013575326529610642}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.19116502155351175, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021818389739288605}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.1287938776528187, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013593155601153983}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.12460529385976211, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016720387661409684}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.21214076270740242, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024214234215899923}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.14557560146488152, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016504135684628225}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.1642477633595285, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.051446930364685284}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b57019c8bd4414a020e6e5d945c394cb211ff4ce --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.12900473919499333, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017118520253344887}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.21188860117767108, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024200971461099297}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.1482921039193071, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016630062554405015}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.019625735698912888, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006788960559737867}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.03345776890551466, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010983051652779822}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.022621289432461413, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006759123203024886}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.0949870967228749, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011613070587285638}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.16170981950543165, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018487736949410018}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.11038136838287661, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011255913731094058}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.12067725065015839, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001582252925566245}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.19897184180815913, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002269910929298289}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.13888020965795267, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001538379368353134}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.0511085425068505, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04359896048820564}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0384d0d912159bb259589a23352378bf470a85c1 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.131296284657946, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018695219758564053}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.21086112976398658, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024798609090401495}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.14876323548663245, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017282343794534887}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.022246438510229897, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000735093927197517}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.03676788461477473, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0012027915912279183}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.02515872989593042, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007219485898619858}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.10120528732821295, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013393173794577398}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.1685999790456406, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020094812074908626}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.11605185407696257, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012554912055135618}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.12222677284783239, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001730583144997117}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.19746168673540157, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023250943809650622}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.13872529165102102, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001595021323222974}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.1660936748998563, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05532845623592181}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..1e3f7951835bd12fb6fe436b6f9b22d69dddc33a --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.11886728882700617, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002117436961902696}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.18301300012408095, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027365960743380834}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.12942102319302837, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018991099063723855}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.020892165067732272, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007093909227597518}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.034255725781094704, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011832923635605066}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.023295998323642722, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007234323946730077}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09394026682495725, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016185789750576213}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.14881026635716532, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022243589892185466}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.10311226464834135, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001438938325893686}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.11079167177233729, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001971644296700457}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.17104157307446946, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002548580661389838}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.12066752443667297, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017584735634078772}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.265340223114637, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05305974419270563}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e66cd334a1c95adb6eab94235023750d7962e89a --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.04347599866942829, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017349739514926302}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.06723054439022214, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002444941071035673}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.045692379866183644, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016245002372787258}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.008532177473335619, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00052322837591612}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.01435573208034991, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008962710642139122}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.009234526462927954, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0005068103534681586}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.035282251910914185, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014152025642593898}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.05528801455229885, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020052884812569384}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.03695285146027073, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012771546682848393}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.04056163792559354, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016324631377389203}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.062318846072973326, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0022623971829829354}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.04240539347591881, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015027461272304392}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.38185114792006436, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.033170133477824944}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.json b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..9a5c276173740e680364354d6c003f4798d9fbe0 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.007323897226172739, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007837754439528186}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.010744411299349098, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0010770185189953045}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.007484594707971953, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0007324479370340896}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0014445753420997747, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00022382223328831132}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.002065495307353092, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0003131650640198061}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0014225676462038027, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00019893350251252884}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.005854388539462018, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006159948133613669}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.008752502975319812, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0008799822787146891}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0059785952299307955, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005701708068931441}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.006728084609279956, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0007272517247844241}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.009922959904673715, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0010052360238980997}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.006853919046022395, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.000671955926205317}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 6.2602080046656e-07, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.5856138592741465e-06}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..b2900ce309fc672efd4d2d3afe21076ead669eab --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 2.289745918558512, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03215563731077313}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.12598084276299099, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001545917960535513}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.22805285561153438, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0018119802616538049}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.1501557279333411, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012086984635718584}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.029166192080389276, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006738001190912294}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.054796348941770226, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001107531688997375}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.036284757350333326, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007336693877513527}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.12261551170813723, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013452096848564365}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.22688024477649193, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018271502433520085}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.14855214947624085, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011952131755131189}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.09502160667174993, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013016944263984458}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.16909882260610437, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0015236310692833888}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.1120504952900659, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0010332544225446578}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d742e97fb2b2894aa458d872370303e1d12945f3 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 8.09449655999987, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13642667438290135}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.44111004479764043, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002953108143020618}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.3608788001975658, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027281702135254194}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.37521944714351063, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022326421185016594}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.178229272386396, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002237921266527968}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.14375556839370257, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018514732874407487}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.1493796439905046, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017386297910845444}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3204380127600031, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0025139836015989456}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.2596384667083823, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021495710442580767}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.27046579362765255, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018134092678886017}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.36070374495918167, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0027619431044086187}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.29348912212402833, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024192093023573833}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3055998269373205, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002066250643227866}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..9deae4fbb43f7d8047aa6fd084dc575895c6befc --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 10.175217193712246, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13784550142039873}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.47053380352601787, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00307016933617562}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.3977041363074254, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002796575988658719}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.408285443364839, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022548958231015724}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.2065199007294722, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0023314616276724786}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.17288562905001834, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002009095211241816}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.17695626939661377, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0018247233828514653}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3437491907504797, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002627735396172612}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.2889416752708342, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022852067063173116}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.29662121267866787, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001894596051870991}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.3871735889716785, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002883545145525451}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.32675621969631036, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025834282219804002}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.33542302368061616, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002172990275172974}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a8de09934a434add5c37095d84c65e0c53514e30 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 10.95899291141005, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.14510219158012166}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4742801284914987, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003029362834508303}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.40537719284095897, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002759798861585915}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4147890519932887, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002200663155159767}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.21341518387404107, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002277413844229496}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.18140418541279082, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0020214773963951015}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.18485958152251725, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001818804374770391}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3501141044124549, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0025673834069702152}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.29761137394334775, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002244285217381524}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3047780378528165, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018606596887839261}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.39342019430732056, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0028110434587027}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.33626385766705214, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025464525887822736}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.34392999985692646, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021211175845010518}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..d279fdd7b138f415c255483feb8e590a8aa57c1f --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.386895901628744, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.17542856183037736}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.47946771760907936, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0030474633497828423}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4098274991176103, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027091680449994387}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4196409268768328, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021710912749222416}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.2208665135289913, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002335065735915771}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.1873078455938463, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002030486080937769}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.19125706434734058, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001843330807550655}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3572037886245874, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002621740024446116}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.3043053686745353, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022672170472949715}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.31153094798198316, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018806036308206858}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.4021613124210754, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0028886954487752506}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.34351473383223446, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002540752491705953}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.35173402770317785, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021391341330935206}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..5413207b0e5da6e91552d85966d7f1d2adcaf0a4 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.617505990076015, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1230152849517545}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.48204944092318114, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003002316588927595}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4147323558875995, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026817560395515382}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4236719164386442, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021260399843472844}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.22352606058310165, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002327913895048718}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.1901827983874871, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001981183792707356}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.19392843172112764, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0018034580517400404}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.361239197186565, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002609836253619501}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.30955097516529423, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022630936427814777}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3161741057473021, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018576737298144322}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.40572501710721487, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002856736023057171}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.34898341882378325, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025334407099668887}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.35638243613939563, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021090678093717823}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_0.json b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e1fa10ee71ca75e376929beff3207c4e8c7f179d --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.12787418085852512, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016809987387128077}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.31608895396065767, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0038258963621472996}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.1797789338541641, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002238242168896514}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.02524260853601397, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008789838314577222}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.06455223256931629, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0022712510338785546}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.03581859183628963, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012387561453567732}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.10110932899185718, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012488319791962006}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2517979369134609, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003007404893650144}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.14238699049842188, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001674527345938325}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.09997007206242994, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013620865297938518}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.24911067799370443, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003284656556324502}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.14083616055450246, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018431251297342697}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.2604853518482184, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11004496191613777}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_1.json b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f9223b7e348cac62d543d80bcd812e50d4e5a4fe --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09957805621029524, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015826328115607507}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.24907309214983692, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037072035709258947}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.14061437622548537, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021469447297238734}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.014422568541693291, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000705059325077753}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.038001558703171984, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0019330906622145496}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.020643546172887276, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010090077210428714}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.08060423822617155, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001187307518511086}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2030927031577155, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0028718024678406067}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11400464096094585, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001613450251458609}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.07920702642342262, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001238725256786352}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.1996696815941679, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003024228260990264}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1120614552219532, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016950121986443255}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.7384958544250524, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06239478619701243}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_2.json b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0691b41553550cc12258b7ca09d2696afb4b5d3f --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09947828286481206, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001505953972124371}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2500088431818419, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0035506754631656327}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.14064933595848825, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020423197913432057}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.01416161272079441, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006522278088063398}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03738802866106956, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018025177125313642}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.020289128041526872, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009355631805434565}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.08029292661970147, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011447483960235392}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.20297587412441118, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0027748052371135007}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11367282291632164, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015540783669632508}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.07910212684644351, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011779242203870924}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.20053999161838518, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0029000703183680947}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.11208735221126831, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016100579741970807}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6540495512512473, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.046396408073435084}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_3.json b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..aad5f2bd5e29dc0414a6876fb7b56a84b4317580 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09928411600328295, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017207049356488255}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2410343352596945, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037379249307775606}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.13704945044919534, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021297649776328433}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.014463719167981467, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007177564843346724}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03626335420954676, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017979395825277924}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.02004702502409466, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009513560186344878}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.08148293344499731, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013752158290751798}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.19861959887433947, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0030136624728142264}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11256696904477134, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016888741184647035}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.08006151730068613, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001397533484888105}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.1956647178062859, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0031019723176881576}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.11061967534852658, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017149674457776968}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6787751322323665, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.057288969732158974}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_4.json b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..9e452324a521dba5aaf4b4fe2af34730c75d1394 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.03382027297391176, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0021686654006393977}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.06188104609121618, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0035631950002271155}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.03944856506020353, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002251645256177216}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0043345633526090405, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004900373009004593}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.009215113275770669, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009737666105924845}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.005582914103565551, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0005973029828213665}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0270009115492642, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017166886306361855}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.04988445629832932, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0028634952961296358}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.031506185461726156, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017733421627202282}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.027542054440807866, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001754002667048103}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.05109342646999838, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002968111419480692}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.032176640736522516, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018223513126861273}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.3128798564689982, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06759192476758741}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_5.json b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..3a7b638a27bfc675b19a4e04a8dec78656e08525 --- /dev/null +++ b/619m22b22b/evaluation/generation/agg.619m22b22b_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0029022502443847146, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007868134866487451}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.002486342324266138, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0006539776334353158}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.002605571931474663, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0006877241279195647}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.00016795311606632358, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 9.692846902628299e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.00011160223424374368, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 6.47826415933139e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0001334632029523884, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 7.71009936644679e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.002165705965212362, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0005672590028000034}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0019339282290457608, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0005063837989061088}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0019829920552960826, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005112934424537438}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0023631872327035596, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0006197355863254781}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0020884836666200664, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.000544462145036003}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0021539578955755926, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005555719062803632}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.111973102172656e-38, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.9314239497663127e-33}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f1a44170aa5df8a265014fdb65588a2a2a5d6d8c --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cd86ff3c842b4d429476ecfd7b95c75538e2bc6ab29a3386631c39c37dc1639 +size 4148737 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..96f769f9e72875b617efd62af566116bed0f1963 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b861ec68b40ba5dceb77eedd734f6fdc20bc2e62402619036fcf4fad849612e7 +size 5106716 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f4738fc49639d466b8f0b404e390b004436f6c87 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30e7c50c6b26b8e58b43cc9d10afd0d18303b6208670e714ed945a6bde1a3e7 +size 6008979 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5eaca71b1d486bbe0765a5f8a39f02cdb8865e3d --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a60f270ad6b7e0748a1c840854e8e71b89a48534aba61534dc224e54daaca18 +size 6911031 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1e53ee233e107fe5ab93ee01c273d3947996daed --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93455e142fd72a158cc56ba00df3d6b00f24b834e6800ea4c3883de4ef99f627 +size 7829366 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ec72f47c1b6a2e092c243591234beb911511fe23 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11914670a65d2066defb267445562246eba78e20b05574f0fe1ad5979d8abeba +size 8726376 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b3aa99a05358815fcaaaf18aeca021a67535e2ca --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc080b9092e69be99af63b2b56cd7e45a269e0b8058ac82ce5679dc794c0f12d +size 7688036 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8e584c46a8adae805dbc6604fe683c515a2b66f2 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35712f080a36eb9224fd8c59e5453905c4d449bfc219af750bebaeb82e51644d +size 13337923 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8b16dd20df1f7836c97d96524349217ed96cae9b --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155c51d9caeeecff7b55ccd09108104e54fbf69e6b34a926c02d8db86f609c25 +size 18925124 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fb2fad589e4c06d7c2b0b7527f55d816e1983bd1 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7b2d473782b4e265d9c896c29a81ab3607730c44fd741bcfc3bf7b067cad859 +size 24339476 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ef8e7c181ff61aca0e2b4d84ebc2a80232870ad4 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08f256d0f62aa664f53986204d9b1a3ca0ff67891e7360da6209d1340dd5d60b +size 29474806 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c07aa610dff73f1798f8946bc7e6f4e79514bd00 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc97861c938c35c563a9c8bcb9ece0d1fb1cd6fb791644abe0d371e3d8561e2 +size 34799776 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3a3d335986ba5f9674bead2e5c6216d6a48a00d2 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8ac3b6727b603a80eea655b273a8bef36a15374af7a8dcbfcfa40f9a48840c6 +size 4429940 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e0d88ac7b859c934624b2976be085ebd77618cc2 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a27a8ed377d10c7b17e3f7a0d6798811ae09fe2b82d22c253f7cee39ddddf97 +size 5044486 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b6f404d3deb95585a4e467578d68ecb4c909aa8f --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fb7c3697c96de76ef43a473fbefdf0bd3c55b1b85fc80998d3fe1e657e699e0 +size 6140041 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e8f211fa603831acca0c2caba9cbd6ed5957e9ce --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b070ec540ba214147020ec602c45d3f0dd885e1c3be28f95ee99edbf406dc742 +size 7225348 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6d7a5c58169996c17276e6b47a8602803b74b261 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4566b8279824100ff427424ab26e8030908e7692b8dde50b4549fe06f11fe04b +size 8304701 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9babc9b66f7c7527c7d9a3ccb23ffaf031db3daa --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77e0011f0a162e02b75199f980f90fabdc6741c92c09aa8eba9d92d44b93bd55 +size 9393944 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_0.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..eacafa14f6ee1365e3f10f83c062d07a26a2038b --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e507e63493249b9bb2154fe4ff4818b041ba8c2380cd600e84020bf2c048e084 +size 2839185 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_1.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b8b955a50eb28cd15f8b1654eee356b9a1bdbecb --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4663d9de2c779b34b28705f09bc485a3df5b21391377df008e16a80528f54fde +size 5109694 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_2.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..495e9f427fee29657e96321e76925702d77dfec7 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:359b76217fe603531c4a82c126f27c31ce9a353adfcb697868a323182a506d98 +size 7382720 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_3.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1edbbf1fdceb508a12048bc0aa2d74198d52863d --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cad5165f55a7094cb2356f42c97bf041122ba2eb164f15df9b533739fcd7a5b6 +size 9649955 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_4.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e8070d74f743d275cb0b22493680ae369cf271e6 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1c61cf59ea3154d47ae940219d35dfd761468217eb32026e762e928204191d5 +size 11673349 diff --git a/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_5.jsonl b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2bdba12e0fe7656affc2bb6c3a017ca99881b128 --- /dev/null +++ b/619m22b22b/evaluation/generation/examples.619m22b22b_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb7895db8e76a0db8a6374fd20d62395483ce5f52d38aac90e5db269b6fbeeaf +size 13897524 diff --git a/619m22b22b/evaluation/generation/merged.csv b/619m22b22b/evaluation/generation/merged.csv new file mode 100644 index 0000000000000000000000000000000000000000..bf1be018b4d6ff7ee4375230e0c4fb805293ed50 --- /dev/null +++ b/619m22b22b/evaluation/generation/merged.csv @@ -0,0 +1,53 @@ +dataset,fewshots,prompt,metric,value +e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.036284757350333326 +e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.036284757350333326 +e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.1493796439905046 +e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.1493796439905046 +e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.17695626939661377 +e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.17695626939661377 +e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.18485958152251725 +e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.18485958152251725 +e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.19125706434734058 +e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.19125706434734058 +e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.19392843172112764 +e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.19392843172112764 +e2e_nlg_cleaned,5,average,multiple,0.15544429138807286 +gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.03581859183628963 +gem_xsum,0,median,rouge2_fmeasure,0.03581859183628963 +gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.020643546172887276 +gem_xsum,1,median,rouge2_fmeasure,0.020643546172887276 +gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.020289128041526872 +gem_xsum,2,median,rouge2_fmeasure,0.020289128041526872 +gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.02004702502409466 +gem_xsum,3,median,rouge2_fmeasure,0.02004702502409466 +gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.005582914103565551 +gem_xsum,4,median,rouge2_fmeasure,0.005582914103565551 +gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.0001334632029523884 +gem_xsum,5,median,rouge2_fmeasure,0.0001334632029523884 +gem_xsum,5,average,multiple,0.01708577806355273 +web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.046357243917834384 +web_nlg_en,0,median,rouge2_fmeasure,0.046357243917834384 +web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.04645784299011615 +web_nlg_en,1,median,rouge2_fmeasure,0.04645784299011615 +web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.04336836166197178 +web_nlg_en,2,median,rouge2_fmeasure,0.04336836166197178 +web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.04336590682538455 +web_nlg_en,3,median,rouge2_fmeasure,0.04336590682538455 +web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.04381547514401627 +web_nlg_en,4,median,rouge2_fmeasure,0.04381547514401627 +web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.04471206930702434 +web_nlg_en,5,median,rouge2_fmeasure,0.04471206930702434 +web_nlg_en,5,average,multiple,0.044679483307724574 +wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.029634443330971802 +wiki_lingua_en,0,median,rouge2_fmeasure,0.029634443330971802 +wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.022621289432461413 +wiki_lingua_en,1,median,rouge2_fmeasure,0.022621289432461413 +wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.02515872989593042 +wiki_lingua_en,2,median,rouge2_fmeasure,0.02515872989593042 +wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.023295998323642722 +wiki_lingua_en,3,median,rouge2_fmeasure,0.023295998323642722 +wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.009234526462927954 +wiki_lingua_en,4,median,rouge2_fmeasure,0.009234526462927954 +wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.0014225676462038027 +wiki_lingua_en,5,median,rouge2_fmeasure,0.0014225676462038027 +wiki_lingua_en,5,average,multiple,0.01856125918202302 diff --git a/619m22b22b/evaluation/generation/merged.json b/619m22b22b/evaluation/generation/merged.json new file mode 100644 index 0000000000000000000000000000000000000000..a49aa360f1d89442b14c6d6590a6e0009bf790c2 --- /dev/null +++ b/619m22b22b/evaluation/generation/merged.json @@ -0,0 +1 @@ +{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.284489408206782, "bleu_stderr": 0.028278761736760676, "rouge1_fmeasure": 0.09675681057276961, "rouge1_fmeasure_stderr": 0.001919696307619028, "rouge1_precision": 0.06358663835483319, "rouge1_precision_stderr": 0.0014889693861394535, "rouge1_recall": 0.27174214833640686, "rouge1_recall_stderr": 0.00457462478813889, "rouge2_fmeasure": 0.046357243917834384, "rouge2_fmeasure_stderr": 0.0012013193200828695, "rouge2_precision": 0.030422997079269397, "rouge2_precision_stderr": 0.0009323098094739095, "rouge2_recall": 0.13467661456157262, "rouge2_recall_stderr": 0.003054061694040572, "rougeL_fmeasure": 0.09413955269824087, "rougeL_fmeasure_stderr": 0.0018351981952839998, "rougeL_precision": 0.061672065465079716, "rougeL_precision_stderr": 0.0014080585643345802, "rougeL_recall": 0.2661377360349057, "rougeL_recall_stderr": 0.004497165496966715, "rougeLsum_fmeasure": 0.09267090154658023, "rougeLsum_fmeasure_stderr": 0.0018160333067075032, "rougeLsum_precision": 0.06091800415353282, "rougeLsum_precision_stderr": 0.0014171331193967864, "rougeLsum_recall": 0.260194740499195, "rougeLsum_recall_stderr": 0.004313457116793654}}, "1": {"PALM_prompt": {"bleu": 0.30794107382055563, "bleu_stderr": 0.025784978769138028, "rouge1_fmeasure": 0.10120740105666726, "rouge1_fmeasure_stderr": 0.0019092721691292443, "rouge1_precision": 0.06665371404806113, "rouge1_precision_stderr": 0.0015734440957952153, "rouge1_recall": 0.3144424936447866, "rouge1_recall_stderr": 0.0051206124313989, "rouge2_fmeasure": 0.04645784299011615, "rouge2_fmeasure_stderr": 0.0011967388539773235, "rouge2_precision": 0.030646030870042525, "rouge2_precision_stderr": 0.001004143407358295, "rouge2_recall": 0.14761105493339954, "rouge2_recall_stderr": 0.0032987706705033693, "rougeL_fmeasure": 0.0965472169328251, "rougeL_fmeasure_stderr": 0.0017612040569045689, "rougeL_precision": 0.06342458988254415, "rougeL_precision_stderr": 0.001438569090801163, "rougeL_recall": 0.3009513725545963, "rougeL_recall_stderr": 0.00484963269043915, "rougeLsum_fmeasure": 0.09616343757930998, "rougeLsum_fmeasure_stderr": 0.0017828648565307438, "rougeLsum_precision": 0.06335872417977906, "rougeLsum_precision_stderr": 0.001469450758465004, "rougeLsum_recall": 0.2974222169844293, "rougeLsum_recall_stderr": 0.004695371476880067}}, "2": {"PALM_prompt": {"bleu": 0.2637530179976743, "bleu_stderr": 0.016909949021643642, "rouge1_fmeasure": 0.09501611598218968, "rouge1_fmeasure_stderr": 0.001741927321610452, "rouge1_precision": 0.0615020828840054, "rouge1_precision_stderr": 0.0013433273947903563, "rouge1_recall": 0.299445798213358, "rouge1_recall_stderr": 0.004915871939991809, "rouge2_fmeasure": 0.04336836166197178, "rouge2_fmeasure_stderr": 0.0010640454351919701, "rouge2_precision": 0.02801441282828333, "rouge2_precision_stderr": 0.0008028230234396205, "rouge2_recall": 0.14194025928903903, "rouge2_recall_stderr": 0.003182936083641798, "rougeL_fmeasure": 0.09086602850993777, "rougeL_fmeasure_stderr": 0.0016091969110186804, "rougeL_precision": 0.05880744102356161, "rougeL_precision_stderr": 0.0012482717064613974, "rougeL_recall": 0.28566216477217093, "rougeL_recall_stderr": 0.004570189393285611, "rougeLsum_fmeasure": 0.09117136970494266, "rougeLsum_fmeasure_stderr": 0.001656189124088108, "rougeLsum_precision": 0.059093493415271564, "rougeLsum_precision_stderr": 0.001289531903153165, "rougeLsum_recall": 0.2860223803179543, "rougeLsum_recall_stderr": 0.004582771242586325}}, "3": {"PALM_prompt": {"bleu": 0.30127710888872544, "bleu_stderr": 0.03017358616348081, "rouge1_fmeasure": 0.0945906963669224, "rouge1_fmeasure_stderr": 0.0017693406808415686, "rouge1_precision": 0.06104007013714008, "rouge1_precision_stderr": 0.0013491551372300523, "rouge1_recall": 0.29887189317521123, "rouge1_recall_stderr": 0.004914239206319181, "rouge2_fmeasure": 0.04336590682538455, "rouge2_fmeasure_stderr": 0.0011190741529048644, "rouge2_precision": 0.027910230506178427, "rouge2_precision_stderr": 0.000841081292987736, "rouge2_recall": 0.1423375012631199, "rouge2_recall_stderr": 0.0032267429123653078, "rougeL_fmeasure": 0.09028317266790117, "rougeL_fmeasure_stderr": 0.0016079490173524383, "rougeL_precision": 0.058159150618774094, "rougeL_precision_stderr": 0.001205587984308903, "rougeL_recall": 0.2849440964479655, "rougeL_recall_stderr": 0.004563998064685871, "rougeLsum_fmeasure": 0.09077654667629809, "rougeLsum_fmeasure_stderr": 0.001652139375900535, "rougeLsum_precision": 0.05857104196241217, "rougeLsum_precision_stderr": 0.0012455598399315733, "rougeLsum_recall": 0.28603952371373775, "rougeLsum_recall_stderr": 0.004599540876569589}}, "4": {"PALM_prompt": {"bleu": 0.2872597163393854, "bleu_stderr": 0.028404283672526617, "rouge1_fmeasure": 0.09539577595993691, "rouge1_fmeasure_stderr": 0.0016905421986277095, "rouge1_precision": 0.06131644179463926, "rouge1_precision_stderr": 0.0012834363684846703, "rouge1_recall": 0.3028350802402193, "rouge1_recall_stderr": 0.0047300632430552995, "rouge2_fmeasure": 0.04381547514401627, "rouge2_fmeasure_stderr": 0.0010528299563527775, "rouge2_precision": 0.028022710412489778, "rouge2_precision_stderr": 0.0007615815376667828, "rouge2_recall": 0.14606820900953005, "rouge2_recall_stderr": 0.003196813879267001, "rougeL_fmeasure": 0.09097440961486976, "rougeL_fmeasure_stderr": 0.0015665857329272954, "rougeL_precision": 0.05845302564855468, "rougeL_precision_stderr": 0.0011796896274871784, "rougeL_recall": 0.2878847835539455, "rougeL_recall_stderr": 0.004378448457675036, "rougeLsum_fmeasure": 0.09161480579427395, "rougeLsum_fmeasure_stderr": 0.0015993125551969124, "rougeLsum_precision": 0.058888194545668245, "rougeLsum_precision_stderr": 0.0012066969569740117, "rougeLsum_recall": 0.29032520701751985, "rougeLsum_recall_stderr": 0.004449008541788328}}, "5": {"PALM_prompt": {"bleu": 0.31000340585989805, "bleu_stderr": 0.035124165599714846, "rouge1_fmeasure": 0.09691863328157169, "rouge1_fmeasure_stderr": 0.0016948022167593932, "rouge1_precision": 0.06197295473456213, "rouge1_precision_stderr": 0.0012498429939017263, "rouge1_recall": 0.30771780339869886, "rouge1_recall_stderr": 0.0047481327408507015, "rouge2_fmeasure": 0.04471206930702434, "rouge2_fmeasure_stderr": 0.0010614115742719097, "rouge2_precision": 0.02844038599993891, "rouge2_precision_stderr": 0.0007609587837791093, "rouge2_recall": 0.15003058591952856, "rouge2_recall_stderr": 0.003250392179692442, "rougeL_fmeasure": 0.09219347050075496, "rougeL_fmeasure_stderr": 0.0015673266760248422, "rougeL_precision": 0.058896059472105716, "rougeL_precision_stderr": 0.001142867644266226, "rougeL_recall": 0.2918380879479885, "rougeL_recall_stderr": 0.0043711606101936385, "rougeLsum_fmeasure": 0.09326972432857124, "rougeLsum_fmeasure_stderr": 0.0016226044008928713, "rougeLsum_precision": 0.05964055672474598, "rougeLsum_precision_stderr": 0.0011914634127819287, "rougeLsum_recall": 0.2955051016060543, "rougeLsum_recall_stderr": 0.004487785833382257}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 1.1642477633595285, "bleu_stderr": 0.051446930364685284, "rouge1_fmeasure": 0.15907081996190248, "rouge1_fmeasure_stderr": 0.0018212459268317244, "rouge1_precision": 0.13630335885749084, "rouge1_precision_stderr": 0.0018434565261148661, "rouge1_recall": 0.23093456995895148, "rouge1_recall_stderr": 0.0026298312792475474, "rouge2_fmeasure": 0.029634443330971802, "rouge2_fmeasure_stderr": 0.000753347585845674, "rouge2_precision": 0.025089156441511235, "rouge2_precision_stderr": 0.0006575369718712219, "rouge2_recall": 0.045617278146314374, "rouge2_recall_stderr": 0.0013397315108689542, "rougeL_fmeasure": 0.1287938776528187, "rougeL_fmeasure_stderr": 0.0013593155601153983, "rougeL_precision": 0.10924764012514825, "rougeL_precision_stderr": 0.0013575326529610642, "rougeL_recall": 0.19116502155351175, "rougeL_recall_stderr": 0.0021818389739288605, "rougeLsum_fmeasure": 0.14557560146488152, "rougeLsum_fmeasure_stderr": 0.0016504135684628225, "rougeLsum_precision": 0.12460529385976211, "rougeLsum_precision_stderr": 0.0016720387661409684, "rougeLsum_recall": 0.21214076270740242, "rougeLsum_recall_stderr": 0.0024214234215899923}}, "1": {"tldr_en": {"bleu": 1.0511085425068505, "bleu_stderr": 0.04359896048820564, "rouge1_fmeasure": 0.1482921039193071, "rouge1_fmeasure_stderr": 0.0016630062554405015, "rouge1_precision": 0.12900473919499333, "rouge1_precision_stderr": 0.0017118520253344887, "rouge1_recall": 0.21188860117767108, "rouge1_recall_stderr": 0.0024200971461099297, "rouge2_fmeasure": 0.022621289432461413, "rouge2_fmeasure_stderr": 0.0006759123203024886, "rouge2_precision": 0.019625735698912888, "rouge2_precision_stderr": 0.0006788960559737867, "rouge2_recall": 0.03345776890551466, "rouge2_recall_stderr": 0.0010983051652779822, "rougeL_fmeasure": 0.11038136838287661, "rougeL_fmeasure_stderr": 0.0011255913731094058, "rougeL_precision": 0.0949870967228749, "rougeL_precision_stderr": 0.0011613070587285638, "rougeL_recall": 0.16170981950543165, "rougeL_recall_stderr": 0.0018487736949410018, "rougeLsum_fmeasure": 0.13888020965795267, "rougeLsum_fmeasure_stderr": 0.001538379368353134, "rougeLsum_precision": 0.12067725065015839, "rougeLsum_precision_stderr": 0.001582252925566245, "rougeLsum_recall": 0.19897184180815913, "rougeLsum_recall_stderr": 0.002269910929298289}}, "2": {"tldr_en": {"bleu": 1.1660936748998563, "bleu_stderr": 0.05532845623592181, "rouge1_fmeasure": 0.14876323548663245, "rouge1_fmeasure_stderr": 0.0017282343794534887, "rouge1_precision": 0.131296284657946, "rouge1_precision_stderr": 0.0018695219758564053, "rouge1_recall": 0.21086112976398658, "rouge1_recall_stderr": 0.0024798609090401495, "rouge2_fmeasure": 0.02515872989593042, "rouge2_fmeasure_stderr": 0.0007219485898619858, "rouge2_precision": 0.022246438510229897, "rouge2_precision_stderr": 0.000735093927197517, "rouge2_recall": 0.03676788461477473, "rouge2_recall_stderr": 0.0012027915912279183, "rougeL_fmeasure": 0.11605185407696257, "rougeL_fmeasure_stderr": 0.0012554912055135618, "rougeL_precision": 0.10120528732821295, "rougeL_precision_stderr": 0.0013393173794577398, "rougeL_recall": 0.1685999790456406, "rougeL_recall_stderr": 0.0020094812074908626, "rougeLsum_fmeasure": 0.13872529165102102, "rougeLsum_fmeasure_stderr": 0.001595021323222974, "rougeLsum_precision": 0.12222677284783239, "rougeLsum_precision_stderr": 0.001730583144997117, "rougeLsum_recall": 0.19746168673540157, "rougeLsum_recall_stderr": 0.0023250943809650622}}, "3": {"tldr_en": {"bleu": 1.265340223114637, "bleu_stderr": 0.05305974419270563, "rouge1_fmeasure": 0.12942102319302837, "rouge1_fmeasure_stderr": 0.0018991099063723855, "rouge1_precision": 0.11886728882700617, "rouge1_precision_stderr": 0.002117436961902696, "rouge1_recall": 0.18301300012408095, "rouge1_recall_stderr": 0.0027365960743380834, "rouge2_fmeasure": 0.023295998323642722, "rouge2_fmeasure_stderr": 0.0007234323946730077, "rouge2_precision": 0.020892165067732272, "rouge2_precision_stderr": 0.0007093909227597518, "rouge2_recall": 0.034255725781094704, "rouge2_recall_stderr": 0.0011832923635605066, "rougeL_fmeasure": 0.10311226464834135, "rougeL_fmeasure_stderr": 0.001438938325893686, "rougeL_precision": 0.09394026682495725, "rougeL_precision_stderr": 0.0016185789750576213, "rougeL_recall": 0.14881026635716532, "rougeL_recall_stderr": 0.0022243589892185466, "rougeLsum_fmeasure": 0.12066752443667297, "rougeLsum_fmeasure_stderr": 0.0017584735634078772, "rougeLsum_precision": 0.11079167177233729, "rougeLsum_precision_stderr": 0.001971644296700457, "rougeLsum_recall": 0.17104157307446946, "rougeLsum_recall_stderr": 0.002548580661389838}}, "4": {"tldr_en": {"bleu": 0.38185114792006436, "bleu_stderr": 0.033170133477824944, "rouge1_fmeasure": 0.045692379866183644, "rouge1_fmeasure_stderr": 0.0016245002372787258, "rouge1_precision": 0.04347599866942829, "rouge1_precision_stderr": 0.0017349739514926302, "rouge1_recall": 0.06723054439022214, "rouge1_recall_stderr": 0.002444941071035673, "rouge2_fmeasure": 0.009234526462927954, "rouge2_fmeasure_stderr": 0.0005068103534681586, "rouge2_precision": 0.008532177473335619, "rouge2_precision_stderr": 0.00052322837591612, "rouge2_recall": 0.01435573208034991, "rouge2_recall_stderr": 0.0008962710642139122, "rougeL_fmeasure": 0.03695285146027073, "rougeL_fmeasure_stderr": 0.0012771546682848393, "rougeL_precision": 0.035282251910914185, "rougeL_precision_stderr": 0.0014152025642593898, "rougeL_recall": 0.05528801455229885, "rougeL_recall_stderr": 0.0020052884812569384, "rougeLsum_fmeasure": 0.04240539347591881, "rougeLsum_fmeasure_stderr": 0.0015027461272304392, "rougeLsum_precision": 0.04056163792559354, "rougeLsum_precision_stderr": 0.0016324631377389203, "rougeLsum_recall": 0.062318846072973326, "rougeLsum_recall_stderr": 0.0022623971829829354}}, "5": {"tldr_en": {"bleu": 6.2602080046656e-07, "bleu_stderr": 1.5856138592741465e-06, "rouge1_fmeasure": 0.007484594707971953, "rouge1_fmeasure_stderr": 0.0007324479370340896, "rouge1_precision": 0.007323897226172739, "rouge1_precision_stderr": 0.0007837754439528186, "rouge1_recall": 0.010744411299349098, "rouge1_recall_stderr": 0.0010770185189953045, "rouge2_fmeasure": 0.0014225676462038027, "rouge2_fmeasure_stderr": 0.00019893350251252884, "rouge2_precision": 0.0014445753420997747, "rouge2_precision_stderr": 0.00022382223328831132, "rouge2_recall": 0.002065495307353092, "rouge2_recall_stderr": 0.0003131650640198061, "rougeL_fmeasure": 0.0059785952299307955, "rougeL_fmeasure_stderr": 0.0005701708068931441, "rougeL_precision": 0.005854388539462018, "rougeL_precision_stderr": 0.0006159948133613669, "rougeL_recall": 0.008752502975319812, "rougeL_recall_stderr": 0.0008799822787146891, "rougeLsum_fmeasure": 0.006853919046022395, "rougeLsum_fmeasure_stderr": 0.000671955926205317, "rougeLsum_precision": 0.006728084609279956, "rougeLsum_precision_stderr": 0.0007272517247844241, "rougeLsum_recall": 0.009922959904673715, "rougeLsum_recall_stderr": 0.0010052360238980997}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 2.289745918558512, "bleu_stderr": 0.03215563731077313, "rouge1_fmeasure": 0.1501557279333411, "rouge1_fmeasure_stderr": 0.0012086984635718584, "rouge1_precision": 0.12598084276299099, "rouge1_precision_stderr": 0.001545917960535513, "rouge1_recall": 0.22805285561153438, "rouge1_recall_stderr": 0.0018119802616538049, "rouge2_fmeasure": 0.036284757350333326, "rouge2_fmeasure_stderr": 0.0007336693877513527, "rouge2_precision": 0.029166192080389276, "rouge2_precision_stderr": 0.0006738001190912294, "rouge2_recall": 0.054796348941770226, "rouge2_recall_stderr": 0.001107531688997375, "rougeL_fmeasure": 0.14855214947624085, "rougeL_fmeasure_stderr": 0.0011952131755131189, "rougeL_precision": 0.12261551170813723, "rougeL_precision_stderr": 0.0013452096848564365, "rougeL_recall": 0.22688024477649193, "rougeL_recall_stderr": 0.0018271502433520085, "rougeLsum_fmeasure": 0.1120504952900659, "rougeLsum_fmeasure_stderr": 0.0010332544225446578, "rougeLsum_precision": 0.09502160667174993, "rougeLsum_precision_stderr": 0.0013016944263984458, "rougeLsum_recall": 0.16909882260610437, "rougeLsum_recall_stderr": 0.0015236310692833888}}, "1": {"generate_text_restaurant": {"bleu": 8.09449655999987, "bleu_stderr": 0.13642667438290135, "rouge1_fmeasure": 0.37521944714351063, "rouge1_fmeasure_stderr": 0.0022326421185016594, "rouge1_precision": 0.44111004479764043, "rouge1_precision_stderr": 0.002953108143020618, "rouge1_recall": 0.3608788001975658, "rouge1_recall_stderr": 0.0027281702135254194, "rouge2_fmeasure": 0.1493796439905046, "rouge2_fmeasure_stderr": 0.0017386297910845444, "rouge2_precision": 0.178229272386396, "rouge2_precision_stderr": 0.002237921266527968, "rouge2_recall": 0.14375556839370257, "rouge2_recall_stderr": 0.0018514732874407487, "rougeL_fmeasure": 0.27046579362765255, "rougeL_fmeasure_stderr": 0.0018134092678886017, "rougeL_precision": 0.3204380127600031, "rougeL_precision_stderr": 0.0025139836015989456, "rougeL_recall": 0.2596384667083823, "rougeL_recall_stderr": 0.0021495710442580767, "rougeLsum_fmeasure": 0.3055998269373205, "rougeLsum_fmeasure_stderr": 0.002066250643227866, "rougeLsum_precision": 0.36070374495918167, "rougeLsum_precision_stderr": 0.0027619431044086187, "rougeLsum_recall": 0.29348912212402833, "rougeLsum_recall_stderr": 0.0024192093023573833}}, "2": {"generate_text_restaurant": {"bleu": 10.175217193712246, "bleu_stderr": 0.13784550142039873, "rouge1_fmeasure": 0.408285443364839, "rouge1_fmeasure_stderr": 0.0022548958231015724, "rouge1_precision": 0.47053380352601787, "rouge1_precision_stderr": 0.00307016933617562, "rouge1_recall": 0.3977041363074254, "rouge1_recall_stderr": 0.002796575988658719, "rouge2_fmeasure": 0.17695626939661377, "rouge2_fmeasure_stderr": 0.0018247233828514653, "rouge2_precision": 0.2065199007294722, "rouge2_precision_stderr": 0.0023314616276724786, "rouge2_recall": 0.17288562905001834, "rouge2_recall_stderr": 0.002009095211241816, "rougeL_fmeasure": 0.29662121267866787, "rougeL_fmeasure_stderr": 0.001894596051870991, "rougeL_precision": 0.3437491907504797, "rougeL_precision_stderr": 0.002627735396172612, "rougeL_recall": 0.2889416752708342, "rougeL_recall_stderr": 0.0022852067063173116, "rougeLsum_fmeasure": 0.33542302368061616, "rougeLsum_fmeasure_stderr": 0.002172990275172974, "rougeLsum_precision": 0.3871735889716785, "rougeLsum_precision_stderr": 0.002883545145525451, "rougeLsum_recall": 0.32675621969631036, "rougeLsum_recall_stderr": 0.0025834282219804002}}, "3": {"generate_text_restaurant": {"bleu": 10.95899291141005, "bleu_stderr": 0.14510219158012166, "rouge1_fmeasure": 0.4147890519932887, "rouge1_fmeasure_stderr": 0.002200663155159767, "rouge1_precision": 0.4742801284914987, "rouge1_precision_stderr": 0.003029362834508303, "rouge1_recall": 0.40537719284095897, "rouge1_recall_stderr": 0.002759798861585915, "rouge2_fmeasure": 0.18485958152251725, "rouge2_fmeasure_stderr": 0.001818804374770391, "rouge2_precision": 0.21341518387404107, "rouge2_precision_stderr": 0.002277413844229496, "rouge2_recall": 0.18140418541279082, "rouge2_recall_stderr": 0.0020214773963951015, "rougeL_fmeasure": 0.3047780378528165, "rougeL_fmeasure_stderr": 0.0018606596887839261, "rougeL_precision": 0.3501141044124549, "rougeL_precision_stderr": 0.0025673834069702152, "rougeL_recall": 0.29761137394334775, "rougeL_recall_stderr": 0.002244285217381524, "rougeLsum_fmeasure": 0.34392999985692646, "rougeLsum_fmeasure_stderr": 0.0021211175845010518, "rougeLsum_precision": 0.39342019430732056, "rougeLsum_precision_stderr": 0.0028110434587027, "rougeLsum_recall": 0.33626385766705214, "rougeLsum_recall_stderr": 0.0025464525887822736}}, "4": {"generate_text_restaurant": {"bleu": 11.386895901628744, "bleu_stderr": 0.17542856183037736, "rouge1_fmeasure": 0.4196409268768328, "rouge1_fmeasure_stderr": 0.0021710912749222416, "rouge1_precision": 0.47946771760907936, "rouge1_precision_stderr": 0.0030474633497828423, "rouge1_recall": 0.4098274991176103, "rouge1_recall_stderr": 0.0027091680449994387, "rouge2_fmeasure": 0.19125706434734058, "rouge2_fmeasure_stderr": 0.001843330807550655, "rouge2_precision": 0.2208665135289913, "rouge2_precision_stderr": 0.002335065735915771, "rouge2_recall": 0.1873078455938463, "rouge2_recall_stderr": 0.002030486080937769, "rougeL_fmeasure": 0.31153094798198316, "rougeL_fmeasure_stderr": 0.0018806036308206858, "rougeL_precision": 0.3572037886245874, "rougeL_precision_stderr": 0.002621740024446116, "rougeL_recall": 0.3043053686745353, "rougeL_recall_stderr": 0.0022672170472949715, "rougeLsum_fmeasure": 0.35173402770317785, "rougeLsum_fmeasure_stderr": 0.0021391341330935206, "rougeLsum_precision": 0.4021613124210754, "rougeLsum_precision_stderr": 0.0028886954487752506, "rougeLsum_recall": 0.34351473383223446, "rougeLsum_recall_stderr": 0.002540752491705953}}, "5": {"generate_text_restaurant": {"bleu": 11.617505990076015, "bleu_stderr": 0.1230152849517545, "rouge1_fmeasure": 0.4236719164386442, "rouge1_fmeasure_stderr": 0.0021260399843472844, "rouge1_precision": 0.48204944092318114, "rouge1_precision_stderr": 0.003002316588927595, "rouge1_recall": 0.4147323558875995, "rouge1_recall_stderr": 0.0026817560395515382, "rouge2_fmeasure": 0.19392843172112764, "rouge2_fmeasure_stderr": 0.0018034580517400404, "rouge2_precision": 0.22352606058310165, "rouge2_precision_stderr": 0.002327913895048718, "rouge2_recall": 0.1901827983874871, "rouge2_recall_stderr": 0.001981183792707356, "rougeL_fmeasure": 0.3161741057473021, "rougeL_fmeasure_stderr": 0.0018576737298144322, "rougeL_precision": 0.361239197186565, "rougeL_precision_stderr": 0.002609836253619501, "rougeL_recall": 0.30955097516529423, "rougeL_recall_stderr": 0.0022630936427814777, "rougeLsum_fmeasure": 0.35638243613939563, "rougeLsum_fmeasure_stderr": 0.0021090678093717823, "rougeLsum_precision": 0.40572501710721487, "rougeLsum_precision_stderr": 0.002856736023057171, "rougeLsum_recall": 0.34898341882378325, "rougeLsum_recall_stderr": 0.0025334407099668887}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 1.2604853518482184, "bleu_stderr": 0.11004496191613777, "rouge1_fmeasure": 0.1797789338541641, "rouge1_fmeasure_stderr": 0.002238242168896514, "rouge1_precision": 0.12787418085852512, "rouge1_precision_stderr": 0.0016809987387128077, "rouge1_recall": 0.31608895396065767, "rouge1_recall_stderr": 0.0038258963621472996, "rouge2_fmeasure": 0.03581859183628963, "rouge2_fmeasure_stderr": 0.0012387561453567732, "rouge2_precision": 0.02524260853601397, "rouge2_precision_stderr": 0.0008789838314577222, "rouge2_recall": 0.06455223256931629, "rouge2_recall_stderr": 0.0022712510338785546, "rougeL_fmeasure": 0.14238699049842188, "rougeL_fmeasure_stderr": 0.001674527345938325, "rougeL_precision": 0.10110932899185718, "rougeL_precision_stderr": 0.0012488319791962006, "rougeL_recall": 0.2517979369134609, "rougeL_recall_stderr": 0.003007404893650144, "rougeLsum_fmeasure": 0.14083616055450246, "rougeLsum_fmeasure_stderr": 0.0018431251297342697, "rougeLsum_precision": 0.09997007206242994, "rougeLsum_precision_stderr": 0.0013620865297938518, "rougeLsum_recall": 0.24911067799370443, "rougeLsum_recall_stderr": 0.003284656556324502}}, "1": {"article_DOC_summary": {"bleu": 0.7384958544250524, "bleu_stderr": 0.06239478619701243, "rouge1_fmeasure": 0.14061437622548537, "rouge1_fmeasure_stderr": 0.0021469447297238734, "rouge1_precision": 0.09957805621029524, "rouge1_precision_stderr": 0.0015826328115607507, "rouge1_recall": 0.24907309214983692, "rouge1_recall_stderr": 0.0037072035709258947, "rouge2_fmeasure": 0.020643546172887276, "rouge2_fmeasure_stderr": 0.0010090077210428714, "rouge2_precision": 0.014422568541693291, "rouge2_precision_stderr": 0.000705059325077753, "rouge2_recall": 0.038001558703171984, "rouge2_recall_stderr": 0.0019330906622145496, "rougeL_fmeasure": 0.11400464096094585, "rougeL_fmeasure_stderr": 0.001613450251458609, "rougeL_precision": 0.08060423822617155, "rougeL_precision_stderr": 0.001187307518511086, "rougeL_recall": 0.2030927031577155, "rougeL_recall_stderr": 0.0028718024678406067, "rougeLsum_fmeasure": 0.1120614552219532, "rougeLsum_fmeasure_stderr": 0.0016950121986443255, "rougeLsum_precision": 0.07920702642342262, "rougeLsum_precision_stderr": 0.001238725256786352, "rougeLsum_recall": 0.1996696815941679, "rougeLsum_recall_stderr": 0.003024228260990264}}, "2": {"article_DOC_summary": {"bleu": 0.6540495512512473, "bleu_stderr": 0.046396408073435084, "rouge1_fmeasure": 0.14064933595848825, "rouge1_fmeasure_stderr": 0.0020423197913432057, "rouge1_precision": 0.09947828286481206, "rouge1_precision_stderr": 0.001505953972124371, "rouge1_recall": 0.2500088431818419, "rouge1_recall_stderr": 0.0035506754631656327, "rouge2_fmeasure": 0.020289128041526872, "rouge2_fmeasure_stderr": 0.0009355631805434565, "rouge2_precision": 0.01416161272079441, "rouge2_precision_stderr": 0.0006522278088063398, "rouge2_recall": 0.03738802866106956, "rouge2_recall_stderr": 0.0018025177125313642, "rougeL_fmeasure": 0.11367282291632164, "rougeL_fmeasure_stderr": 0.0015540783669632508, "rougeL_precision": 0.08029292661970147, "rougeL_precision_stderr": 0.0011447483960235392, "rougeL_recall": 0.20297587412441118, "rougeL_recall_stderr": 0.0027748052371135007, "rougeLsum_fmeasure": 0.11208735221126831, "rougeLsum_fmeasure_stderr": 0.0016100579741970807, "rougeLsum_precision": 0.07910212684644351, "rougeLsum_precision_stderr": 0.0011779242203870924, "rougeLsum_recall": 0.20053999161838518, "rougeLsum_recall_stderr": 0.0029000703183680947}}, "3": {"article_DOC_summary": {"bleu": 0.6787751322323665, "bleu_stderr": 0.057288969732158974, "rouge1_fmeasure": 0.13704945044919534, "rouge1_fmeasure_stderr": 0.0021297649776328433, "rouge1_precision": 0.09928411600328295, "rouge1_precision_stderr": 0.0017207049356488255, "rouge1_recall": 0.2410343352596945, "rouge1_recall_stderr": 0.0037379249307775606, "rouge2_fmeasure": 0.02004702502409466, "rouge2_fmeasure_stderr": 0.0009513560186344878, "rouge2_precision": 0.014463719167981467, "rouge2_precision_stderr": 0.0007177564843346724, "rouge2_recall": 0.03626335420954676, "rouge2_recall_stderr": 0.0017979395825277924, "rougeL_fmeasure": 0.11256696904477134, "rougeL_fmeasure_stderr": 0.0016888741184647035, "rougeL_precision": 0.08148293344499731, "rougeL_precision_stderr": 0.0013752158290751798, "rougeL_recall": 0.19861959887433947, "rougeL_recall_stderr": 0.0030136624728142264, "rougeLsum_fmeasure": 0.11061967534852658, "rougeLsum_fmeasure_stderr": 0.0017149674457776968, "rougeLsum_precision": 0.08006151730068613, "rougeLsum_precision_stderr": 0.001397533484888105, "rougeLsum_recall": 0.1956647178062859, "rougeLsum_recall_stderr": 0.0031019723176881576}}, "4": {"article_DOC_summary": {"bleu": 0.3128798564689982, "bleu_stderr": 0.06759192476758741, "rouge1_fmeasure": 0.03944856506020353, "rouge1_fmeasure_stderr": 0.002251645256177216, "rouge1_precision": 0.03382027297391176, "rouge1_precision_stderr": 0.0021686654006393977, "rouge1_recall": 0.06188104609121618, "rouge1_recall_stderr": 0.0035631950002271155, "rouge2_fmeasure": 0.005582914103565551, "rouge2_fmeasure_stderr": 0.0005973029828213665, "rouge2_precision": 0.0043345633526090405, "rouge2_precision_stderr": 0.0004900373009004593, "rouge2_recall": 0.009215113275770669, "rouge2_recall_stderr": 0.0009737666105924845, "rougeL_fmeasure": 0.031506185461726156, "rougeL_fmeasure_stderr": 0.0017733421627202282, "rougeL_precision": 0.0270009115492642, "rougeL_precision_stderr": 0.0017166886306361855, "rougeL_recall": 0.04988445629832932, "rougeL_recall_stderr": 0.0028634952961296358, "rougeLsum_fmeasure": 0.032176640736522516, "rougeLsum_fmeasure_stderr": 0.0018223513126861273, "rougeLsum_precision": 0.027542054440807866, "rougeLsum_precision_stderr": 0.001754002667048103, "rougeLsum_recall": 0.05109342646999838, "rougeLsum_recall_stderr": 0.002968111419480692}}, "5": {"article_DOC_summary": {"bleu": 2.111973102172656e-38, "bleu_stderr": 2.9314239497663127e-33, "rouge1_fmeasure": 0.002605571931474663, "rouge1_fmeasure_stderr": 0.0006877241279195647, "rouge1_precision": 0.0029022502443847146, "rouge1_precision_stderr": 0.0007868134866487451, "rouge1_recall": 0.002486342324266138, "rouge1_recall_stderr": 0.0006539776334353158, "rouge2_fmeasure": 0.0001334632029523884, "rouge2_fmeasure_stderr": 7.71009936644679e-05, "rouge2_precision": 0.00016795311606632358, "rouge2_precision_stderr": 9.692846902628299e-05, "rouge2_recall": 0.00011160223424374368, "rouge2_recall_stderr": 6.47826415933139e-05, "rougeL_fmeasure": 0.0019829920552960826, "rougeL_fmeasure_stderr": 0.0005112934424537438, "rougeL_precision": 0.002165705965212362, "rougeL_precision_stderr": 0.0005672590028000034, "rougeL_recall": 0.0019339282290457608, "rougeL_recall_stderr": 0.0005063837989061088, "rougeLsum_fmeasure": 0.0021539578955755926, "rougeLsum_fmeasure_stderr": 0.0005555719062803632, "rougeLsum_precision": 0.0023631872327035596, "rougeLsum_precision_stderr": 0.0006197355863254781, "rougeLsum_recall": 0.0020884836666200664, "rougeLsum_recall_stderr": 0.000544462145036003}}}} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ad5655a3116c0622c0bcaa43e8c8c1d050344559 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.284489408206782, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.028278761736760676 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06358663835483319, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014889693861394535 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.27174214833640686, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.00457462478813889 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09675681057276961, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001919696307619028 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.030422997079269397, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0009323098094739095 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13467661456157262, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003054061694040572 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.046357243917834384, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0012013193200828695 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.061672065465079716, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014080585643345802 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2661377360349057, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004497165496966715 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09413955269824087, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0018351981952839998 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06091800415353282, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0014171331193967864 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.260194740499195, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004313457116793654 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09267090154658023, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018160333067075032 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..3ef12efa47fcb0cd8c28a9d31f4d392f44c1cd40 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.30794107382055563, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.025784978769138028 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06665371404806113, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0015734440957952153 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.3144424936447866, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0051206124313989 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.10120740105666726, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019092721691292443 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.030646030870042525, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.001004143407358295 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.14761105493339954, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0032987706705033693 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04645784299011615, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011967388539773235 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06342458988254415, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001438569090801163 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.3009513725545963, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.00484963269043915 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.0965472169328251, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0017612040569045689 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06335872417977906, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001469450758465004 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2974222169844293, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004695371476880067 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09616343757930998, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017828648565307438 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e6de4ba813fb327b4af9eda4c86c3ab1be661570 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.2637530179976743, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.016909949021643642 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.0615020828840054, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0013433273947903563 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.299445798213358, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004915871939991809 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09501611598218968, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001741927321610452 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.02801441282828333, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008028230234396205 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.14194025928903903, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003182936083641798 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04336836166197178, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010640454351919701 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05880744102356161, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012482717064613974 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.28566216477217093, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004570189393285611 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09086602850993777, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016091969110186804 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.059093493415271564, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001289531903153165 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2860223803179543, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004582771242586325 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09117136970494266, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001656189124088108 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..bddd9a05e10024792a3e81dc8b14bc643b46c860 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.30127710888872544, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03017358616348081 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06104007013714008, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0013491551372300523 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.29887189317521123, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004914239206319181 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.0945906963669224, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017693406808415686 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.027910230506178427, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.000841081292987736 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1423375012631199, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0032267429123653078 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04336590682538455, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011190741529048644 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.058159150618774094, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001205587984308903 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2849440964479655, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004563998064685871 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09028317266790117, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016079490173524383 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05857104196241217, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012455598399315733 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.28603952371373775, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004599540876569589 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09077654667629809, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001652139375900535 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e100e1d160223f696abf9c61713e660e8950751e --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.2872597163393854, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.028404283672526617 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06131644179463926, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0012834363684846703 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.3028350802402193, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0047300632430552995 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09539577595993691, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016905421986277095 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.028022710412489778, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007615815376667828 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.14606820900953005, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003196813879267001 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04381547514401627, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010528299563527775 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05845302564855468, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011796896274871784 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2878847835539455, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004378448457675036 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09097440961486976, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015665857329272954 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.058888194545668245, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012066969569740117 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.29032520701751985, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004449008541788328 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09161480579427395, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015993125551969124 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..fc72ec2e77adc24ccf46072838c50d03686c4964 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.31000340585989805, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.035124165599714846 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06197295473456213, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0012498429939017263 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.30771780339869886, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0047481327408507015 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09691863328157169, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016948022167593932 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.02844038599993891, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007609587837791093 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.15003058591952856, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003250392179692442 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04471206930702434, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0010614115742719097 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.058896059472105716, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001142867644266226 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2918380879479885, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0043711606101936385 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09219347050075496, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0015673266760248422 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05964055672474598, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0011914634127819287 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2955051016060543, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004487785833382257 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09326972432857124, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016226044008928713 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9c8ce981ebe622fb5258898e87e3184350333ae9 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.13630335885749084, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0018434565261148661 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.23093456995895148, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0026298312792475474 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.15907081996190248, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018212459268317244 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.025089156441511235, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006575369718712219 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.045617278146314374, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0013397315108689542 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.029634443330971802, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.000753347585845674 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.10924764012514825, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013575326529610642 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.19116502155351175, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0021818389739288605 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.1287938776528187, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013593155601153983 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.12460529385976211, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016720387661409684 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.21214076270740242, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0024214234215899923 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.14557560146488152, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016504135684628225 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.1642477633595285, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.051446930364685284 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..4743725d140ca3b0229ab46bd2ce19d0ebb1056e --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.12900473919499333, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017118520253344887 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.21188860117767108, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0024200971461099297 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.1482921039193071, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016630062554405015 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.019625735698912888, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006788960559737867 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.03345776890551466, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0010983051652779822 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.022621289432461413, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0006759123203024886 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.0949870967228749, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011613070587285638 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.16170981950543165, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0018487736949410018 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.11038136838287661, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0011255913731094058 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.12067725065015839, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001582252925566245 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.19897184180815913, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002269910929298289 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.13888020965795267, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001538379368353134 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.0511085425068505, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.04359896048820564 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0c50ea6af6c9c6ba84a5745dc673f8849d69b278 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.131296284657946, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0018695219758564053 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.21086112976398658, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0024798609090401495 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.14876323548663245, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0017282343794534887 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.022246438510229897, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.000735093927197517 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.03676788461477473, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0012027915912279183 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.02515872989593042, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0007219485898619858 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.10120528732821295, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013393173794577398 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.1685999790456406, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0020094812074908626 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.11605185407696257, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0012554912055135618 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.12222677284783239, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001730583144997117 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.19746168673540157, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0023250943809650622 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.13872529165102102, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001595021323222974 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.1660936748998563, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05532845623592181 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f59678b545c0f799e7f0923dbb18370c05cc26be --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.11886728882700617, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002117436961902696 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.18301300012408095, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0027365960743380834 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.12942102319302837, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018991099063723855 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.020892165067732272, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0007093909227597518 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.034255725781094704, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0011832923635605066 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.023295998323642722, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0007234323946730077 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.09394026682495725, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0016185789750576213 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.14881026635716532, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0022243589892185466 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.10311226464834135, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001438938325893686 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.11079167177233729, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001971644296700457 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.17104157307446946, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002548580661389838 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.12066752443667297, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017584735634078772 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.265340223114637, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05305974419270563 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..bad5fb44c7865adf527223bfadde87252c97d10d --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.04347599866942829, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017349739514926302 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.06723054439022214, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002444941071035673 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.045692379866183644, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016245002372787258 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.008532177473335619, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00052322837591612 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.01435573208034991, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0008962710642139122 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.009234526462927954, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0005068103534681586 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.035282251910914185, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014152025642593898 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.05528801455229885, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0020052884812569384 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.03695285146027073, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0012771546682848393 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.04056163792559354, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016324631377389203 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.062318846072973326, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0022623971829829354 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.04240539347591881, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015027461272304392 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.38185114792006436, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.033170133477824944 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.json b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..ffdceb22b752a08cf4fffd513059a9848c0015a4 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.007323897226172739, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0007837754439528186 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.010744411299349098, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0010770185189953045 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.007484594707971953, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0007324479370340896 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.0014445753420997747, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00022382223328831132 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.002065495307353092, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0003131650640198061 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.0014225676462038027, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00019893350251252884 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.005854388539462018, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0006159948133613669 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.008752502975319812, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0008799822787146891 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0059785952299307955, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0005701708068931441 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.006728084609279956, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0007272517247844241 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.009922959904673715, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0010052360238980997 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.006853919046022395, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.000671955926205317 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 6.2602080046656e-07, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 1.5856138592741465e-06 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8d8406f2a480fc7684718c93d2f4d7b6f77e333a --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 2.289745918558512, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.03215563731077313 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.12598084276299099, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.001545917960535513 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.22805285561153438, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0018119802616538049 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.1501557279333411, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0012086984635718584 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.029166192080389276, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0006738001190912294 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.054796348941770226, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.001107531688997375 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.036284757350333326, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0007336693877513527 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.12261551170813723, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0013452096848564365 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.22688024477649193, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0018271502433520085 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.14855214947624085, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0011952131755131189 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.09502160667174993, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0013016944263984458 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.16909882260610437, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0015236310692833888 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.1120504952900659, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0010332544225446578 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..88d1db0df68f36e6ddb5c3415100f1af76cde6d6 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 8.09449655999987, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.13642667438290135 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.44111004479764043, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002953108143020618 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.3608788001975658, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027281702135254194 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.37521944714351063, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0022326421185016594 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.178229272386396, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002237921266527968 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.14375556839370257, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0018514732874407487 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.1493796439905046, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017386297910845444 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3204380127600031, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0025139836015989456 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.2596384667083823, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0021495710442580767 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.27046579362765255, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018134092678886017 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.36070374495918167, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0027619431044086187 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.29348912212402833, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0024192093023573833 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3055998269373205, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002066250643227866 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..6c87d71257b20e8b125143523f07a02af206ce66 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 10.175217193712246, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.13784550142039873 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.47053380352601787, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.00307016933617562 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.3977041363074254, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002796575988658719 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.408285443364839, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0022548958231015724 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.2065199007294722, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0023314616276724786 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.17288562905001834, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.002009095211241816 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.17695626939661377, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0018247233828514653 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3437491907504797, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002627735396172612 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.2889416752708342, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022852067063173116 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.29662121267866787, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.001894596051870991 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.3871735889716785, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002883545145525451 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.32675621969631036, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025834282219804002 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.33542302368061616, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002172990275172974 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..52cae758b8d1c016681d3a9cb4bad6c6c9b1f687 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 10.95899291141005, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.14510219158012166 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4742801284914987, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003029362834508303 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.40537719284095897, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002759798861585915 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4147890519932887, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002200663155159767 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.21341518387404107, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002277413844229496 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.18140418541279082, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0020214773963951015 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.18485958152251725, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.001818804374770391 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3501141044124549, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0025673834069702152 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.29761137394334775, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002244285217381524 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3047780378528165, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018606596887839261 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.39342019430732056, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0028110434587027 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.33626385766705214, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025464525887822736 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.34392999985692646, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0021211175845010518 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..c699431544b5afcade36448f1de465054d7bfee3 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.386895901628744, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.17542856183037736 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.47946771760907936, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0030474633497828423 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4098274991176103, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0027091680449994387 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4196409268768328, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0021710912749222416 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.2208665135289913, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002335065735915771 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.1873078455938463, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.002030486080937769 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.19125706434734058, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.001843330807550655 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3572037886245874, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002621740024446116 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.3043053686745353, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022672170472949715 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.31153094798198316, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018806036308206858 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.4021613124210754, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0028886954487752506 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.34351473383223446, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002540752491705953 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.35173402770317785, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0021391341330935206 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..2b73a134167cdc52dd46ee557cc4c839fb6cdc5e --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.617505990076015, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.1230152849517545 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.48204944092318114, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003002316588927595 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4147323558875995, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0026817560395515382 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4236719164386442, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0021260399843472844 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.22352606058310165, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002327913895048718 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.1901827983874871, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.001981183792707356 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.19392843172112764, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0018034580517400404 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.361239197186565, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002609836253619501 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.30955097516529423, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022630936427814777 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3161741057473021, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018576737298144322 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.40572501710721487, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002856736023057171 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.34898341882378325, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0025334407099668887 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.35638243613939563, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0021090678093717823 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_0.json b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..49c18ec970271bd3afe2f01fe360bd7be9ca848f --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.12787418085852512, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0016809987387128077 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.31608895396065767, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0038258963621472996 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.1797789338541641, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002238242168896514 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.02524260853601397, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0008789838314577222 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.06455223256931629, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0022712510338785546 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.03581859183628963, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0012387561453567732 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.10110932899185718, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0012488319791962006 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2517979369134609, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.003007404893650144 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.14238699049842188, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.001674527345938325 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.09997007206242994, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0013620865297938518 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.24911067799370443, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003284656556324502 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.14083616055450246, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0018431251297342697 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.2604853518482184, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.11004496191613777 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_1.json b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..23a76e76eaaa9d88e0b3e6fdd955dc11251d0bed --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.09957805621029524, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0015826328115607507 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.24907309214983692, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0037072035709258947 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.14061437622548537, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0021469447297238734 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.014422568541693291, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.000705059325077753 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.038001558703171984, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0019330906622145496 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.020643546172887276, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0010090077210428714 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.08060423822617155, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001187307518511086 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2030927031577155, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0028718024678406067 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11400464096094585, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.001613450251458609 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.07920702642342262, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001238725256786352 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.1996696815941679, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003024228260990264 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1120614552219532, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016950121986443255 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.7384958544250524, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.06239478619701243 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_2.json b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..1c09f554234f2586390211eca781c856bce57871 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.09947828286481206, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.001505953972124371 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2500088431818419, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0035506754631656327 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.14064933595848825, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0020423197913432057 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.01416161272079441, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006522278088063398 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03738802866106956, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0018025177125313642 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.020289128041526872, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009355631805434565 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.08029292661970147, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0011447483960235392 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.20297587412441118, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0027748052371135007 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11367282291632164, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0015540783669632508 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.07910212684644351, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0011779242203870924 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.20053999161838518, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0029000703183680947 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.11208735221126831, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016100579741970807 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.6540495512512473, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.046396408073435084 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_3.json b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..de9364bf0d1913b31b96285eba040bbd65c80425 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.09928411600328295, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0017207049356488255 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2410343352596945, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0037379249307775606 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.13704945044919534, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0021297649776328433 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.014463719167981467, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0007177564843346724 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03626335420954676, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0017979395825277924 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.02004702502409466, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009513560186344878 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.08148293344499731, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0013752158290751798 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.19861959887433947, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0030136624728142264 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11256696904477134, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016888741184647035 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.08006151730068613, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001397533484888105 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.1956647178062859, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0031019723176881576 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.11061967534852658, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0017149674457776968 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.6787751322323665, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.057288969732158974 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_4.json b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..6b70037f07f477c6db0eebd700235b8ec8cd9b0a --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.03382027297391176, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0021686654006393977 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.06188104609121618, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0035631950002271155 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.03944856506020353, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002251645256177216 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0043345633526090405, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0004900373009004593 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.009215113275770669, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0009737666105924845 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.005582914103565551, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0005973029828213665 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0270009115492642, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0017166886306361855 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.04988445629832932, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0028634952961296358 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.031506185461726156, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0017733421627202282 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.027542054440807866, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001754002667048103 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.05109342646999838, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.002968111419480692 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.032176640736522516, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0018223513126861273 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.3128798564689982, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.06759192476758741 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_5.json b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..5c1ca361fd6fc208b44c463db15155cdc44c6b50 --- /dev/null +++ b/619m22b22b/evaluation/generation/slim.619m22b22b_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0029022502443847146, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0007868134866487451 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.002486342324266138, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0006539776334353158 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.002605571931474663, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0006877241279195647 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.00016795311606632358, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 9.692846902628299e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.00011160223424374368, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 6.47826415933139e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0001334632029523884, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 7.71009936644679e-05 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.002165705965212362, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0005672590028000034 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.0019339282290457608, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0005063837989061088 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.0019829920552960826, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0005112934424537438 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.0023631872327035596, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0006197355863254781 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.0020884836666200664, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.000544462145036003 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0021539578955755926, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0005555719062803632 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 2.111973102172656e-38, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 2.9314239497663127e-33 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b22b/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_0.csv b/619m22b22b/evaluation/rankeval/619m22b22b_0.csv new file mode 100644 index 0000000000000000000000000000000000000000..81a618d33f740df5c9553d1876bdd142b13dac9e --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_0.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.327,0.01484221315341124,0 +anli_r2,acc,0.339,0.014976758771620345,0 +anli_r3,acc,0.32916666666666666,0.013570806258433618,0 +arc_challenge,acc,0.2022184300341297,0.011737454431872105,0 +arc_challenge,acc_norm,0.25597269624573377,0.012753013241244516,0 +arc_easy,acc,0.4978956228956229,0.01025969265153704,0 +arc_easy,acc_norm,0.43392255892255893,0.010169795770462115,0 +boolq,acc,0.5565749235474006,0.008688893661318222,1 +cb,acc,0.48214285714285715,0.0673769750864465,1 +cb,f1,0.3354385964912281,,1 +copa,acc,0.68,0.046882617226215034,0 +hellaswag,acc,0.33658633738299143,0.004715762925037023,0 +hellaswag,acc_norm,0.3966341366261701,0.004881990487628906,0 +piqa,acc,0.6800870511425462,0.010882873582092056,0 +piqa,acc_norm,0.691512513601741,0.010776164678037155,0 +rte,acc,0.5415162454873647,0.029992535385373314,0 +sciq,acc,0.765,0.01341472903024713,0 +sciq,acc_norm,0.668,0.014899597242811471,0 +storycloze_2016,acc,0.6296098343132015,0.01116720970729423,0 +winogrande,acc,0.516179952644041,0.014045126130978601,0 diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_0.json b/619m22b22b/evaluation/rankeval/619m22b22b_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a16b5afdd0a107700cfcb66d98eb19ecf26c5c3b --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.327, + "acc_stderr": 0.01484221315341124 + }, + "anli_r2": { + "acc": 0.339, + "acc_stderr": 0.014976758771620345 + }, + "anli_r3": { + "acc": 0.32916666666666666, + "acc_stderr": 0.013570806258433618 + }, + "cb": { + "acc": 0.48214285714285715, + "acc_stderr": 0.0673769750864465, + "f1": 0.3354385964912281 + }, + "copa": { + "acc": 0.68, + "acc_stderr": 0.046882617226215034 + }, + "hellaswag": { + "acc": 0.33658633738299143, + "acc_stderr": 0.004715762925037023, + "acc_norm": 0.3966341366261701, + "acc_norm_stderr": 0.004881990487628906 + }, + "rte": { + "acc": 0.5415162454873647, + "acc_stderr": 0.029992535385373314 + }, + "winogrande": { + "acc": 0.516179952644041, + "acc_stderr": 0.014045126130978601 + }, + "storycloze_2016": { + "acc": 0.6296098343132015, + "acc_stderr": 0.01116720970729423 + }, + "boolq": { + "acc": 0.5565749235474006, + "acc_stderr": 0.008688893661318222 + }, + "arc_easy": { + "acc": 0.4978956228956229, + "acc_stderr": 0.01025969265153704, + "acc_norm": 0.43392255892255893, + "acc_norm_stderr": 0.010169795770462115 + }, + "arc_challenge": { + "acc": 0.2022184300341297, + "acc_stderr": 0.011737454431872105, + "acc_norm": 0.25597269624573377, + "acc_norm_stderr": 0.012753013241244516 + }, + "sciq": { + "acc": 0.765, + "acc_stderr": 0.01341472903024713, + "acc_norm": 0.668, + "acc_norm_stderr": 0.014899597242811471 + }, + "piqa": { + "acc": 0.6800870511425462, + "acc_stderr": 0.010882873582092056, + "acc_norm": 0.691512513601741, + "acc_norm_stderr": 0.010776164678037155 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_1.csv b/619m22b22b/evaluation/rankeval/619m22b22b_1.csv new file mode 100644 index 0000000000000000000000000000000000000000..29446261ebfd30d24253a1700802ca036b570f9e --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_1.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.339,0.014976758771620344,0 +anli_r2,acc,0.348,0.01507060460376841,0 +anli_r3,acc,0.3308333333333333,0.013588208070708997,0 +arc_challenge,acc,0.20136518771331058,0.011718927477444274,0 +arc_challenge,acc_norm,0.24573378839590443,0.012581033453730114,0 +arc_easy,acc,0.48947811447811446,0.010257511546488225,0 +arc_easy,acc_norm,0.4414983164983165,0.01018931438274994,0 +boolq,acc,0.5480122324159021,0.00870464385117752,1 +cb,acc,0.5357142857142857,0.06724777654937658,1 +cb,f1,0.3734827264239029,,1 +copa,acc,0.68,0.04688261722621505,0 +hellaswag,acc,0.33519219279028084,0.004710928569985744,0 +hellaswag,acc_norm,0.3979286994622585,0.004884702412456095,0 +piqa,acc,0.6800870511425462,0.010882873582092058,0 +piqa,acc_norm,0.6855277475516867,0.010833009065106565,0 +rte,acc,0.5451263537906137,0.029973636495415252,0 +sciq,acc,0.783,0.01304151375727071,0 +sciq,acc_norm,0.718,0.014236526215291336,0 +storycloze_2016,acc,0.6231961517904864,0.011205964516279665,0 +winogrande,acc,0.5035516969218626,0.014052131146915848,0 diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_1.json b/619m22b22b/evaluation/rankeval/619m22b22b_1.json new file mode 100644 index 0000000000000000000000000000000000000000..14d3c8155c2deb322256eda0a8bffb0e21b8e0b7 --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.339, + "acc_stderr": 0.014976758771620344 + }, + "anli_r2": { + "acc": 0.348, + "acc_stderr": 0.01507060460376841 + }, + "anli_r3": { + "acc": 0.3308333333333333, + "acc_stderr": 0.013588208070708997 + }, + "cb": { + "acc": 0.5357142857142857, + "acc_stderr": 0.06724777654937658, + "f1": 0.3734827264239029 + }, + "copa": { + "acc": 0.68, + "acc_stderr": 0.04688261722621505 + }, + "hellaswag": { + "acc": 0.33519219279028084, + "acc_stderr": 0.004710928569985744, + "acc_norm": 0.3979286994622585, + "acc_norm_stderr": 0.004884702412456095 + }, + "rte": { + "acc": 0.5451263537906137, + "acc_stderr": 0.029973636495415252 + }, + "winogrande": { + "acc": 0.5035516969218626, + "acc_stderr": 0.014052131146915848 + }, + "storycloze_2016": { + "acc": 0.6231961517904864, + "acc_stderr": 0.011205964516279665 + }, + "boolq": { + "acc": 0.5480122324159021, + "acc_stderr": 0.00870464385117752 + }, + "arc_easy": { + "acc": 0.48947811447811446, + "acc_stderr": 0.010257511546488225, + "acc_norm": 0.4414983164983165, + "acc_norm_stderr": 0.01018931438274994 + }, + "arc_challenge": { + "acc": 0.20136518771331058, + "acc_stderr": 0.011718927477444274, + "acc_norm": 0.24573378839590443, + "acc_norm_stderr": 0.012581033453730114 + }, + "sciq": { + "acc": 0.783, + "acc_stderr": 0.01304151375727071, + "acc_norm": 0.718, + "acc_norm_stderr": 0.014236526215291336 + }, + "piqa": { + "acc": 0.6800870511425462, + "acc_stderr": 0.010882873582092058, + "acc_norm": 0.6855277475516867, + "acc_norm_stderr": 0.010833009065106565 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_2.csv b/619m22b22b/evaluation/rankeval/619m22b22b_2.csv new file mode 100644 index 0000000000000000000000000000000000000000..49021dea95c29e9b092c86a801573b81b30be7b6 --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_2.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.335,0.014933117490932573,0 +anli_r2,acc,0.33,0.014876872027456736,0 +anli_r3,acc,0.32,0.013471620929769137,0 +arc_challenge,acc,0.19880546075085323,0.011662850198175543,0 +arc_challenge,acc_norm,0.24658703071672355,0.012595726268790118,0 +arc_easy,acc,0.5025252525252525,0.010259652668783464,0 +arc_easy,acc_norm,0.4701178451178451,0.010241444322886428,0 +boolq,acc,0.5382262996941896,0.008719460098106854,1 +cb,acc,0.5357142857142857,0.06724777654937658,1 +cb,f1,0.3734265734265734,,1 +copa,acc,0.62,0.04878317312145632,0 +hellaswag,acc,0.329416450906194,0.004690407826933901,0 +hellaswag,acc_norm,0.39832702648874724,0.004885529674958319,0 +piqa,acc,0.6800870511425462,0.010882873582092056,0 +piqa,acc_norm,0.6779107725788901,0.010902341695103438,0 +rte,acc,0.5523465703971119,0.029931070362939526,0 +sciq,acc,0.795,0.012772554096113118,0 +sciq,acc_norm,0.741,0.013860415257527911,0 +storycloze_2016,acc,0.6253340459647247,0.011193279518329866,0 +winogrande,acc,0.5232833464877664,0.014037241309573642,0 diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_2.json b/619m22b22b/evaluation/rankeval/619m22b22b_2.json new file mode 100644 index 0000000000000000000000000000000000000000..b69f30585e65c5047bc84cbd5b9bc9b9d81362b2 --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.335, + "acc_stderr": 0.014933117490932573 + }, + "anli_r2": { + "acc": 0.33, + "acc_stderr": 0.014876872027456736 + }, + "anli_r3": { + "acc": 0.32, + "acc_stderr": 0.013471620929769137 + }, + "cb": { + "acc": 0.5357142857142857, + "acc_stderr": 0.06724777654937658, + "f1": 0.3734265734265734 + }, + "copa": { + "acc": 0.62, + "acc_stderr": 0.04878317312145632 + }, + "hellaswag": { + "acc": 0.329416450906194, + "acc_stderr": 0.004690407826933901, + "acc_norm": 0.39832702648874724, + "acc_norm_stderr": 0.004885529674958319 + }, + "rte": { + "acc": 0.5523465703971119, + "acc_stderr": 0.029931070362939526 + }, + "winogrande": { + "acc": 0.5232833464877664, + "acc_stderr": 0.014037241309573642 + }, + "storycloze_2016": { + "acc": 0.6253340459647247, + "acc_stderr": 0.011193279518329866 + }, + "boolq": { + "acc": 0.5382262996941896, + "acc_stderr": 0.008719460098106854 + }, + "arc_easy": { + "acc": 0.5025252525252525, + "acc_stderr": 0.010259652668783464, + "acc_norm": 0.4701178451178451, + "acc_norm_stderr": 0.010241444322886428 + }, + "arc_challenge": { + "acc": 0.19880546075085323, + "acc_stderr": 0.011662850198175543, + "acc_norm": 0.24658703071672355, + "acc_norm_stderr": 0.012595726268790118 + }, + "sciq": { + "acc": 0.795, + "acc_stderr": 0.012772554096113118, + "acc_norm": 0.741, + "acc_norm_stderr": 0.013860415257527911 + }, + "piqa": { + "acc": 0.6800870511425462, + "acc_stderr": 0.010882873582092056, + "acc_norm": 0.6779107725788901, + "acc_norm_stderr": 0.010902341695103438 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_3.csv b/619m22b22b/evaluation/rankeval/619m22b22b_3.csv new file mode 100644 index 0000000000000000000000000000000000000000..cc0ce3dc9b5185a5e29590250dcb9c29fcee502a --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_3.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.33,0.014876872027456734,0 +anli_r2,acc,0.344,0.015029633724408945,0 +anli_r3,acc,0.3358333333333333,0.01363926119093288,0 +arc_challenge,acc,0.20477815699658702,0.011792544338513398,0 +arc_challenge,acc_norm,0.2440273037542662,0.012551447627856253,0 +arc_easy,acc,0.4936868686868687,0.01025896566804443,0 +arc_easy,acc_norm,0.4671717171717172,0.010237645778853851,0 +boolq,acc,0.5467889908256881,0.00870668126587249,1 +cb,acc,0.4107142857142857,0.06633634150359541,1 +cb,f1,0.2837426900584795,,1 +copa,acc,0.7,0.046056618647183814,0 +hellaswag,acc,0.33419637522405893,0.004707447244200623,0 +hellaswag,acc_norm,0.3986257717586138,0.0048861479076274025,0 +piqa,acc,0.6822633297062024,0.010863133246569285,0 +piqa,acc_norm,0.6844396082698585,0.010843119201758936,0 +rte,acc,0.5451263537906137,0.029973636495415252,0 +sciq,acc,0.776,0.01319083007236447,0 +sciq,acc_norm,0.745,0.013790038620872844,0 +storycloze_2016,acc,0.6274719401389631,0.01118036151820015,0 +winogrande,acc,0.5138121546961326,0.014047122916440415,0 diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_3.json b/619m22b22b/evaluation/rankeval/619m22b22b_3.json new file mode 100644 index 0000000000000000000000000000000000000000..6ea546210ec6f8f826efb50c81afc1bbc014300f --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.33, + "acc_stderr": 0.014876872027456734 + }, + "anli_r2": { + "acc": 0.344, + "acc_stderr": 0.015029633724408945 + }, + "anli_r3": { + "acc": 0.3358333333333333, + "acc_stderr": 0.01363926119093288 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.06633634150359541, + "f1": 0.2837426900584795 + }, + "copa": { + "acc": 0.7, + "acc_stderr": 0.046056618647183814 + }, + "hellaswag": { + "acc": 0.33419637522405893, + "acc_stderr": 0.004707447244200623, + "acc_norm": 0.3986257717586138, + "acc_norm_stderr": 0.0048861479076274025 + }, + "rte": { + "acc": 0.5451263537906137, + "acc_stderr": 0.029973636495415252 + }, + "winogrande": { + "acc": 0.5138121546961326, + "acc_stderr": 0.014047122916440415 + }, + "storycloze_2016": { + "acc": 0.6274719401389631, + "acc_stderr": 0.01118036151820015 + }, + "boolq": { + "acc": 0.5467889908256881, + "acc_stderr": 0.00870668126587249 + }, + "arc_easy": { + "acc": 0.4936868686868687, + "acc_stderr": 0.01025896566804443, + "acc_norm": 0.4671717171717172, + "acc_norm_stderr": 0.010237645778853851 + }, + "arc_challenge": { + "acc": 0.20477815699658702, + "acc_stderr": 0.011792544338513398, + "acc_norm": 0.2440273037542662, + "acc_norm_stderr": 0.012551447627856253 + }, + "sciq": { + "acc": 0.776, + "acc_stderr": 0.01319083007236447, + "acc_norm": 0.745, + "acc_norm_stderr": 0.013790038620872844 + }, + "piqa": { + "acc": 0.6822633297062024, + "acc_stderr": 0.010863133246569285, + "acc_norm": 0.6844396082698585, + "acc_norm_stderr": 0.010843119201758936 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_4.csv b/619m22b22b/evaluation/rankeval/619m22b22b_4.csv new file mode 100644 index 0000000000000000000000000000000000000000..7ab0f868aa3fbb272ff38e188826b4cd8a7b4ec9 --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_4.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.333,0.014910846164229868,0 +anli_r2,acc,0.339,0.014976758771620344,0 +anli_r3,acc,0.33166666666666667,0.013596836729485159,0 +arc_challenge,acc,0.20648464163822525,0.011828865619002316,0 +arc_challenge,acc_norm,0.24232081911262798,0.012521593295800113,0 +arc_easy,acc,0.4970538720538721,0.010259605416237579,0 +arc_easy,acc_norm,0.4734848484848485,0.010245347015573706,0 +boolq,acc,0.5599388379204893,0.008681991497133588,1 +cb,acc,0.44642857142857145,0.06703189227942398,1 +cb,f1,0.3107226107226107,,1 +copa,acc,0.7,0.046056618647183814,0 +hellaswag,acc,0.3355905198167696,0.004712314511950964,0 +hellaswag,acc_norm,0.39693288189603665,0.004882619484166607,0 +piqa,acc,0.6877040261153428,0.010812581599154424,0 +piqa,acc_norm,0.6849836779107725,0.010838072746240652,0 +rte,acc,0.555956678700361,0.029907396333795987,0 +sciq,acc,0.789,0.012909130321042095,0 +sciq,acc_norm,0.758,0.013550631705555958,0 +storycloze_2016,acc,0.6189203634420096,0.011230638649246914,0 +winogrande,acc,0.5169692186266772,0.014044390401612976,0 diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_4.json b/619m22b22b/evaluation/rankeval/619m22b22b_4.json new file mode 100644 index 0000000000000000000000000000000000000000..185f3d514e82da5f78bec31320482f6f9a031a18 --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_4.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.333, + "acc_stderr": 0.014910846164229868 + }, + "anli_r2": { + "acc": 0.339, + "acc_stderr": 0.014976758771620344 + }, + "anli_r3": { + "acc": 0.33166666666666667, + "acc_stderr": 0.013596836729485159 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.06703189227942398, + "f1": 0.3107226107226107 + }, + "copa": { + "acc": 0.7, + "acc_stderr": 0.046056618647183814 + }, + "hellaswag": { + "acc": 0.3355905198167696, + "acc_stderr": 0.004712314511950964, + "acc_norm": 0.39693288189603665, + "acc_norm_stderr": 0.004882619484166607 + }, + "rte": { + "acc": 0.555956678700361, + "acc_stderr": 0.029907396333795987 + }, + "winogrande": { + "acc": 0.5169692186266772, + "acc_stderr": 0.014044390401612976 + }, + "storycloze_2016": { + "acc": 0.6189203634420096, + "acc_stderr": 0.011230638649246914 + }, + "boolq": { + "acc": 0.5599388379204893, + "acc_stderr": 0.008681991497133588 + }, + "arc_easy": { + "acc": 0.4970538720538721, + "acc_stderr": 0.010259605416237579, + "acc_norm": 0.4734848484848485, + "acc_norm_stderr": 0.010245347015573706 + }, + "arc_challenge": { + "acc": 0.20648464163822525, + "acc_stderr": 0.011828865619002316, + "acc_norm": 0.24232081911262798, + "acc_norm_stderr": 0.012521593295800113 + }, + "sciq": { + "acc": 0.789, + "acc_stderr": 0.012909130321042095, + "acc_norm": 0.758, + "acc_norm_stderr": 0.013550631705555958 + }, + "piqa": { + "acc": 0.6877040261153428, + "acc_stderr": 0.010812581599154424, + "acc_norm": 0.6849836779107725, + "acc_norm_stderr": 0.010838072746240652 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_5.csv b/619m22b22b/evaluation/rankeval/619m22b22b_5.csv new file mode 100644 index 0000000000000000000000000000000000000000..6ea0aab3d19965b0f07fb52b71266f7458124271 --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_5.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.337,0.0149550879186536,0 +anli_r2,acc,0.332,0.014899597242811478,0 +anli_r3,acc,0.33666666666666667,0.0136476029424064,0 +arc_challenge,acc,0.20819112627986347,0.011864866118448072,0 +arc_challenge,acc_norm,0.24914675767918087,0.012639407111926435,0 +arc_easy,acc,0.5122053872053872,0.010256726235129004,0 +arc_easy,acc_norm,0.4772727272727273,0.01024917909060598,0 +boolq,acc,0.5458715596330275,0.008708174519488063,1 +cb,acc,0.44642857142857145,0.06703189227942398,1 +cb,f1,0.30537634408602155,,1 +copa,acc,0.67,0.04725815626252607,0 +hellaswag,acc,0.3339972117108146,0.004706748152125318,0 +hellaswag,acc_norm,0.3973312089225254,0.004883455188908967,0 +piqa,acc,0.6795429815016322,0.010887766073814876,0 +piqa,acc_norm,0.6866158868335147,0.010822829929195475,0 +rte,acc,0.5523465703971119,0.029931070362939526,0 +sciq,acc,0.801,0.012631649083099186,0 +sciq,acc_norm,0.763,0.013454070462577938,0 +storycloze_2016,acc,0.6258685195082844,0.011190071893524303,0 +winogrande,acc,0.5185477505919495,0.014042813708888378,0 diff --git a/619m22b22b/evaluation/rankeval/619m22b22b_5.json b/619m22b22b/evaluation/rankeval/619m22b22b_5.json new file mode 100644 index 0000000000000000000000000000000000000000..2e6dff05fc4a362348254d883b5dcee6fdb55cdf --- /dev/null +++ b/619m22b22b/evaluation/rankeval/619m22b22b_5.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.337, + "acc_stderr": 0.0149550879186536 + }, + "anli_r2": { + "acc": 0.332, + "acc_stderr": 0.014899597242811478 + }, + "anli_r3": { + "acc": 0.33666666666666667, + "acc_stderr": 0.0136476029424064 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.06703189227942398, + "f1": 0.30537634408602155 + }, + "copa": { + "acc": 0.67, + "acc_stderr": 0.04725815626252607 + }, + "hellaswag": { + "acc": 0.3339972117108146, + "acc_stderr": 0.004706748152125318, + "acc_norm": 0.3973312089225254, + "acc_norm_stderr": 0.004883455188908967 + }, + "rte": { + "acc": 0.5523465703971119, + "acc_stderr": 0.029931070362939526 + }, + "winogrande": { + "acc": 0.5185477505919495, + "acc_stderr": 0.014042813708888378 + }, + "storycloze_2016": { + "acc": 0.6258685195082844, + "acc_stderr": 0.011190071893524303 + }, + "boolq": { + "acc": 0.5458715596330275, + "acc_stderr": 0.008708174519488063 + }, + "arc_easy": { + "acc": 0.5122053872053872, + "acc_stderr": 0.010256726235129004, + "acc_norm": 0.4772727272727273, + "acc_norm_stderr": 0.01024917909060598 + }, + "arc_challenge": { + "acc": 0.20819112627986347, + "acc_stderr": 0.011864866118448072, + "acc_norm": 0.24914675767918087, + "acc_norm_stderr": 0.012639407111926435 + }, + "sciq": { + "acc": 0.801, + "acc_stderr": 0.012631649083099186, + "acc_norm": 0.763, + "acc_norm_stderr": 0.013454070462577938 + }, + "piqa": { + "acc": 0.6795429815016322, + "acc_stderr": 0.010887766073814876, + "acc_norm": 0.6866158868335147, + "acc_norm_stderr": 0.010822829929195475 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f85385593b4af63d1123ee65a975564e95e1c44 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f80c606f078c0ecc0cb1fae065ff4f529a1f3fae402c525d0fa405a3c53725b0 +size 116013719 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e215360e2525b5355ee62645f025255a877c94aa --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e211d5e7ea394e8e1cf411abc367e46f60049fa8ff5845125c6bc5cc05acd7df +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99fdf32d2645b8068fb2f0b89c6b6a82cb33263e --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91c16ba73df527af025219fcafb3b1e994f8606da29c736b31224493dd296fef +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c290cf92b61a02c2bbc18115364422f3c8101b6d --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b2810bccbd72481010d1234e0396d732682af7fd1ac54c9affb8b7e32ac19d +size 116013602 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1484e51f9ba12d3b90c18877d4c323fc4089d0e9 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ea606abe68534f56c393b99692def1e909279484f0ceb9c8efd13e0fbb3a0c2 +size 116013858 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bac53505f0d4635c2e3d560a4d4af61b5e7042a6 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e60f63ebfd65cfdd5d0ce1a5c117379f6c41ff601c32a3361dc7defa9adf1ea3 +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7bc707067b0368c9e9512673f5d86679c176194 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f10303ed52af9002d2ba4f3e3c818ac30feff9630ed192fa48cda950dac9181 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c600d691c633885ed347f027eba3731f0202495a --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e5093f3da5a7cced48a3f97e536b3d36d668cbb4672097f285621eff188e5b3 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39832c4bc6abaa6493eb8c6b81770b9b91c55ed2 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53cff2327426ebf81646c7a794fc3eaa9a09c4c10d143a6eb0f71748b670de66 +size 116013602 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..849417b3c2a10dfa61fa08da08e0f803471d32af --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88721b5e44a998a47fc1d8b5044bc206dba8a04a98ed134ae518a62f0114ff44 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c17f3a144b3b285c4f76299853bbeb864844d42 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc1f9ffb63630eb05998ed9cab90281ff2b2d214b5f4a4f4c00eaf4e6ef3b93 +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a56b337f679706994cc172f46a15b524bb9fc64b --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0999a644c15481823f1f8df58fcba45eae9180064170312701c72fa68d33536f +size 116013719 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d567abc8131070536491bd04a826cd15479dd7c --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605a25f11f4257189c3ca2fd4faae45e0d0873b1fa43208f7b58c0d571980e3a +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b88047502f9f095e990a406a451bbf311cf2426 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02057856ff50677a97244e3125a10333431aab7e07d329316c10ae7fae428f11 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5641cffb9ebedb775fd7cf895f6acfbb17e7ce0 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4f04b5bce8b172d3e031c9caadad1193af0c97467deabaf4b22199f9ebee2fe +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af9740a9b60056a11371bcb9efb3df5361f5de02 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dab56743bc53ef8d9fc411ec677979374468af0a615aabe347a841a516b6de1 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe15bad31bc27315e033e0fb3dea680fe18884f4 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df72d0d5c720ae98b587061ba1265f957c46b48280fc034aaab2e86b668e1f6f +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31571a35c9f28e21365b09a6d085191e2f5a330c --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c55e4d928279e2c0a48dac35c00a9bc296391b78676038d27baaaf050984e2ee +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3762fedee32d56cee626b91f79cca2386fbd2414 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde479f995304e4c26e9f554ca13422fe400f7ea79f02eb76d37321d531024e5 +size 116013858 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04db325f6ffbc87d6f15edf680393907c3c53c7b --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56b6b76aa119d1c1cb99cecdb2c269710f72171d28ec0682e2600d97d10e6c7c +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5b9157b7c3037a2df2c0b1284401e12cd0eca41 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8a74baeabae52e103f83d2845c9c5f7e1e7d011a16fff14350f4f234c2c058 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7487bbcf685a9d11aa180801be07a450ea14ac20 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9f084ea3c8297df3d1fcb90f2c0c647806a981c3247e08bdeac27796a5e61dc +size 116013602 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e64cb209cee4676e6bddaf88ce0dddeee720ad6 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10cc31cebd5c6096f8e7b4bf9f85241029c74dc662e8a5b6d85d39cf9c289a95 +size 116013655 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3c343430c1bedde4619f1981edd70f91e892337 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38561a6b0e310cf529825cfebfc76d410b09b5ce401665575893a56533e41a19 +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63a613ed8c91e433d3821237209f5c8c47ce0089 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b368eb55cc5fab0feb51f08a0cc97332c0617e3797f33b08c6171fde3bfb4e0 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22b5a074eb30635570a8cf1a17f86ed47b438ed1 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60b5557f35c22a80be9e68b2d94904da9e110cd88b80c50829eb69dde6534eeb +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f0b2e3ea3e363dd818a3edf34737a03f02745c9 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0343984e620c5583a9db81b94834a8199b99eef8c1c5546aa0f02d3dddc492 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..270313ca4943b39aca7f5b3f6e4603ddb653b7f2 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:946536386829739443926dd47a5c8262533708a483d58b3f92b328bfb333f63a +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e571132649ec60d6033dcd7433a11d76513fe3d1 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81bce925a6e5ac4a11db997645c114c0be635bcc8617d2d1eef3e2ac0a224e60 +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7045151a3d1ec173638b9b0c846c9ae42b757d7c --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b46089fcfb4f501111e544cd64a3ba2185bbdea082a8de2e5c3494902c1eace9 +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..791880c17dd0730f3836e8597d7d5fb331caedd5 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f80affe5660934c01c63cf7ffe848e6466398ecde393fe11125d855be62d34 +size 116013858 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0583949f918c14a1767e0bc52007dc4d53b7eabf --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfed8a07210475fd8bfb0effa751c84c2c3715ce0a9dc095646ec5fe9d2c1cbb +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d510ce59ac369d6c39b552f02c1bcdeccaa8129 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e71efbf2db51067c4984631c45ad6e22b0057449c0e2691c356fca04c756ee65 +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..819481c846483b5ad443127d90b41af3705c5029 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc1cd3d69b1526190eda4860ec0596bc88ebb7c8813d98fd50d645c8cd4052d9 +size 116013719 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..510024cf5b31ced0778de18a01b9101bbb8dc890 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dcc850332da120fe5df915f5ed6edd79a9fcb5ca0f00b814f0fe4b84b194a8a +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20626f041de421728dee81553e4354fdd8f842ae --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e69ebb72680ccb532bceff31683703cb539a8a9b6e4720d099bb6d86ef4cefc +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbc597e6b58210302759c75034ef031f17b41c2a --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1092667277318706ee0ba9e7faa2ffe12bead8754aceee8d214404d813e7cd5b +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6aa05cf82cd3650f95a81ba4f80021073b54d26 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db8d91777c528befe43929259a776f9e279011d2a4140d9131d42e1c8b852e11 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..791edfb729ee210fdef6fd90702c01b9dc2c76b2 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd38c8a39a82cff5f35fc8fa79c1fc19c76c9e50d357495aed2f7daf40a056cb +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34efd1d03630a38ed6c0b6a9f898a6acaec6f19c --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08e3a30c4240f9fe2c4618df8b27c9fe0501e0e25f4ad57f8e203170e98fdf64 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..011bcc1f29185671542ea061dc70e7ff239f328e --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9efc217e8882a0539eb092a2b482b4e1db6ac39bee1f414062578ae40b006189 +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5deeced0a2c627038be8374131f233c4e96b4c96 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c65a4f73c2a241c64c3424faf607db82c84b05895a9fb0ce5055937674fff868 +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21c07d18425163f51a92e62ecc3c1a7d99f514bc --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab7ad53cedbcd7326ddcd805baa0ed188f8b1fca0921e0f9522b147ae0e84894 +size 116013922 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5cf2b5e28b6d4e24b99205697d78665a86b960a6 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d70a637bf437af3693b342c833ac9a880563f5fc6faca0a28fff3d5a2de83be4 +size 116013538 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..007fe75aca8f01c6796a62c461d9cf292b0e3862 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82af0b735fee16690fd2eb39d5ec97f7fb3f0adaff81870f6577e6c5ed4a648d +size 116013719 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27c9f1d338592cb5817eaa7884d1c0610aefbc30 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:050c90825eb806790e15bdbcbe9d99ed2c2030902b3b47a57ffa2cefc74b182a +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6752eeeb116ac0cab618a6dfd0d5439a6290b46d --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffa61885644a981a59a569e2b4dc2b8642a9608a2b09dc81cb53e830c24b4e18 +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18f02770c870513b7bc8f8b86bb19dfaba9b0e62 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b9b73b88327656c9fd4f6fac9da445ce167b91ca02cfd75d3642b0f253ae5e3 +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41632783f89e966617f0cbe823f775c04919841f --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bfb9f89519d1b544250729002994b111fbf735a9bcb37a1eb32de03b39d6d0a +size 116013922 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a1c5c05a476100e4a3f33584dc044126d51faa5 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6e01c62ba2c0d20fac5d6a6e35fde1f39002b0d4fffb7b5b64964982c99a4b8 +size 116013602 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57421b11ffba0600ae75f94a484faf44d6d20bdd --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7dbe34351e59260b1d85ed23998229a9841903ece72ee7c90b298adcda9e8da +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16d605dc6bc61ca6e884db5b6f50c11881183337 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:172efaa5379ff2f10822c7082ee1cb965d6f553d2c470cefedc3981a3fd5d3e0 +size 116013602 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16d8b8a1bfd982ce4438e12e0645cd9043abe681 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d93b6bea2ed143b05bea3a2d2a18a670f098a0eaacc4bdac07a31221e4be8b4 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..947bc6ad0b77874be76a68b85ceff6ccdf22cf3f --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54969e242809c9aeb2535238c5e726ec9ba3273a2d0ab48c77a9ce2f32dd83b3 +size 116013858 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..980f0cfc326e701b1abe0b016f7b8aa16e584834 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8289d528767c5b91194140bae5d984434e594c3314aad8cb724da7c31e2f49a8 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2e51eb03d83c179d97850c35a24687b55e88162 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7af64cfaafa9431acd08c84ef5bb16ed591fbe27fd9621e0dbeb6cdc22dc408d +size 116013655 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2a7d5ca207b773a88988d55a4980aac9001898c --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e4d3af331f6291127d2cf85188a270366561a63a478b7016fba9df51fa9e8c9 +size 116013730 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a97d10d25f83b06c99cce071f96f1d68546c11c5 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82c7e751b2e307d1761bf2e5a975a106bf3f2f9c60a616c85c907e57f29169e0 +size 116013794 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15b9497f5b34f7d1f885578ac9ab72500de26e2a --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e891627ad029e574856be18181f784080bafa372445c779258d958b314eee6c +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f5a0e0d2f91d0c888d4cf144d170e61e96e5e52 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b10f05a4b4be2f3322531dbea5718972948a478701b54b840e8e78b104e43c +size 116013666 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..719216bb23f0b7df2a9d086605ff8bc9104ecb78 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85bb709bd76c4dc0d4173de28453c411bd899baeb70a3076698ab7647fe0847a +size 116013719 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ec7ffb64721f4eb640581c5a708c874bc9271a2 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d4565ac12afee08bd0458e6bde53cebeff6bd123b7b69227a621409e94bf4c9 +size 116013719 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abc55512552da8545f6aed4a5704cac621e686d7 --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0825d38cc27678424f6112f7bbc25571b6fe03a56d3d7813d65a0c33a1ff519b +size 116013655 diff --git a/619m22b22b/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/619m22b22b/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..858ed7532d6bee9b9915eb73d0e156ec771041ba --- /dev/null +++ b/619m22b22b/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f011e107893be0763639336944254364e7857e8164a682cbcd0795766c664e1c +size 116013591 diff --git a/619m22b22b/global_step41007/layer_01-model_00-model_states.pt b/619m22b22b/global_step41007/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b101aaef608fe22cc19c031d098bed4a5a684dca --- /dev/null +++ b/619m22b22b/global_step41007/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f6ba6aa9a9b481106cbfa01c6b650aa5a93c9f27a3a82b447152eb7ac73c571 +size 160826627 diff --git a/619m22b22b/global_step41007/layer_03-model_00-model_states.pt b/619m22b22b/global_step41007/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b3bec429b979572fa0c81ad7c6dede01d5d8595 --- /dev/null +++ b/619m22b22b/global_step41007/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e856dbaa2dbc25f3f6077d4d161c29d811980229100c5a6eac5367b9642acc43 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_04-model_00-model_states.pt b/619m22b22b/global_step41007/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfc9c88c4fbadae41522296ff043b769317a0928 --- /dev/null +++ b/619m22b22b/global_step41007/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b69842a2097792c2d654be6ca90a2142b1c4e36c07661bc9d9f464094db1579 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_05-model_00-model_states.pt b/619m22b22b/global_step41007/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..361ba2516095991c73a3b141efb31615011cffaf --- /dev/null +++ b/619m22b22b/global_step41007/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ac72231342f3e93c6665db49bfb751101228dd6bb16e56e6ceb8a13acf00bf8 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_06-model_00-model_states.pt b/619m22b22b/global_step41007/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d69d662f151b2b4a4ccfd86847731f34e860ffe --- /dev/null +++ b/619m22b22b/global_step41007/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53382585b76aa73963c7538b233c25b45e3fdf2a458aa56d61007c0beb1abcce +size 56667395 diff --git a/619m22b22b/global_step41007/layer_07-model_00-model_states.pt b/619m22b22b/global_step41007/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f4873bd5bbee3ea00b275a79e7faa4df3340da8 --- /dev/null +++ b/619m22b22b/global_step41007/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e555cdd0ed785e07a600652c7112fc4c48e13aa7c123c2b1eaf2dcec0a2ce86 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_08-model_00-model_states.pt b/619m22b22b/global_step41007/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa495599bbccd6c283931921da730f76a06aefb2 --- /dev/null +++ b/619m22b22b/global_step41007/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1ff609e9a4dc346c22786f37e556271482745a6ccfe6021986cf64cc4cde857 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_09-model_00-model_states.pt b/619m22b22b/global_step41007/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f35327e7f8168664c8e43d4b730ec98afe458260 --- /dev/null +++ b/619m22b22b/global_step41007/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ecc545b2831410fdd70c16d0663c8a934178455cee900c9153314e241caa6cf +size 56667395 diff --git a/619m22b22b/global_step41007/layer_10-model_00-model_states.pt b/619m22b22b/global_step41007/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..707f00ee3eac3beafa65e77c333d80d0fd9de3bf --- /dev/null +++ b/619m22b22b/global_step41007/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d08c5b490e165ad5c25edac4ab9b5dfb0131835b9e4e88810a96d884645ad4 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_11-model_00-model_states.pt b/619m22b22b/global_step41007/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3411d52828b5b20436558c0dc786fcd3488833c8 --- /dev/null +++ b/619m22b22b/global_step41007/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b61d81169eef5e147eb88a1fd5107fb077598a6fbfc5cae45b07f474b812274a +size 56667395 diff --git a/619m22b22b/global_step41007/layer_12-model_00-model_states.pt b/619m22b22b/global_step41007/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d85c82fe6928151e971767ff4ee71d5b93d6cee --- /dev/null +++ b/619m22b22b/global_step41007/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c989458af8cbcc7e9de1778445b46257f12fb54f07b7b2a8368b9f4e133c3b43 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_13-model_00-model_states.pt b/619m22b22b/global_step41007/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09610a4367684fdb0b30ed1c0d116376eb5c284d --- /dev/null +++ b/619m22b22b/global_step41007/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4de91736a05599cef0742622f2abcd4e8700515c34c74467a10acf884ab27134 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_14-model_00-model_states.pt b/619m22b22b/global_step41007/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef8f7f952b294f5529f233e012f5599481c3d8d9 --- /dev/null +++ b/619m22b22b/global_step41007/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ff78b1e7bbebd12dd4c1e37a7958fc6a99998cb59ed416d6258ff1d937d78f +size 56667395 diff --git a/619m22b22b/global_step41007/layer_15-model_00-model_states.pt b/619m22b22b/global_step41007/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38ab6aa134511204fe7a2a00ed42de87a48871f1 --- /dev/null +++ b/619m22b22b/global_step41007/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc40943c8aaf99220293ffef7915ed97b1cb1af59e1b8eeeae864e29c9213ec +size 56667395 diff --git a/619m22b22b/global_step41007/layer_16-model_00-model_states.pt b/619m22b22b/global_step41007/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8143bbf9b2a88eb3aff05c184dc6558d711660a --- /dev/null +++ b/619m22b22b/global_step41007/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937a15d9e41f65e8fc6fe47327de1490f25af844b4f792b17a18e492f78c2b54 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_17-model_00-model_states.pt b/619m22b22b/global_step41007/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cadffc19746a4b2056fac984b10e125437eeb9b --- /dev/null +++ b/619m22b22b/global_step41007/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24ca3abb186c1475a44324bcef93574a9bad2febb2d3050dc2bd53817ad2b031 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_18-model_00-model_states.pt b/619m22b22b/global_step41007/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1dfcbc84e28fffe573524398b90b92392a2f5b5d --- /dev/null +++ b/619m22b22b/global_step41007/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:023b64d24d41ff30c07ae3344ac4653cda93796230d05c988dcd1405f804ad98 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_19-model_00-model_states.pt b/619m22b22b/global_step41007/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..407c9ddf27c7ee49821f197d6e1aba351e486616 --- /dev/null +++ b/619m22b22b/global_step41007/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e5fb7db11fff9776ebb377ed66c19661dfd2d2ce6b903e74798b1ae9122687 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_20-model_00-model_states.pt b/619m22b22b/global_step41007/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb6e715abf2b796785d66a5f5c1b6ed4e886fcf2 --- /dev/null +++ b/619m22b22b/global_step41007/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dbfa53e18a47d95981220603344e8237c1b918b303c7ca36923cb1c52cf27ef +size 56667395 diff --git a/619m22b22b/global_step41007/layer_21-model_00-model_states.pt b/619m22b22b/global_step41007/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63e94f0fc87e0b039e47d8b49d91588600c2b2bf --- /dev/null +++ b/619m22b22b/global_step41007/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:399a021c63ae1f764479977791aa97c74d6ef4fb878ec30fb6a32d5a7eab3a48 +size 56667395 diff --git a/619m22b22b/global_step41007/layer_23-model_00-model_states.pt b/619m22b22b/global_step41007/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09993f3a2d32bccd341c293a90512df180167167 --- /dev/null +++ b/619m22b22b/global_step41007/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0de7e80e5598023c6a4de03058535feb6eb15ca1708c7e6adc83dc35937a7d +size 7363 diff --git a/619m22b22b/global_step41007/mp_rank_00_model_states.pt b/619m22b22b/global_step41007/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..140a0f2bc3ca34129236ca95d466548d7af11fc5 --- /dev/null +++ b/619m22b22b/global_step41007/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0722c7e667d305703e2a78ef50784ba2b18fa02bafedca4ea715ed34ae047ad +size 38515 diff --git a/619m22b22b/logs/2817193.err b/619m22b22b/logs/2817193.err new file mode 100644 index 0000000000000000000000000000000000000000..df55d4083114847ac5a2693efdb63028d8797525 --- /dev/null +++ b/619m22b22b/logs/2817193.err @@ -0,0 +1,1117 @@ +0: 2023-02-09 15:26:04.281742: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:04.281750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.296861: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.296876: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:04.320672: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.338548: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:04.364603: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.373214: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:04.411023: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.425546: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:04.453955: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.469847: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:04.498027: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.520265: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:04.542139: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:26:04.551774: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571823: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571827: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571832: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571828: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571836: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571839: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571822: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:26:04.571845: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588862: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588866: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588877: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588868: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588878: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588864: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588869: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:26:04.588889: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599404: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599407: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599400: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599416: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599413: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599417: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599424: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:26:04.599427: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605834: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605838: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605838: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605841: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605848: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605830: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605855: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:26:04.605860: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619484: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619483: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619492: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619494: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619491: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619490: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619492: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:26:04.619498: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622791: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622792: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622790: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622796: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622784: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622786: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:26:04.622811: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:26:17.299930: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.299943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.299961: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.300263: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:26:17.299981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.299997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.300280: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:26:17.300284: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:26:17.300010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.299989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.300001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:26:17.300296: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:26:17.300304: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:26:17.300312: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:26:17.300315: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:26:17.300316: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372072: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372073: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:26:17.372816: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372820: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372827: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372830: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372832: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372830: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:26:17.372837: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425056: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425082: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425086: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:26:17.425458: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425464: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425468: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425469: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425472: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425471: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425473: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:26:17.425496: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557527: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557525: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557535: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557705: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557541: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557709: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557543: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:26:17.557713: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557717: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557717: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557724: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557727: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:26:17.557728: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559334: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559339: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559339: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:26:17.559562: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559568: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559569: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559570: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559574: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559574: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559578: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:26:17.559586: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.941897: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.941902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.941908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.941910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.941917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.941912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.941912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.942102: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.941899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:26:17.942109: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.942112: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.942113: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.942116: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.942118: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.942120: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:26:17.942121: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.540619: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540937: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.540642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540780: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540966: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.540663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540640: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540976: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.540983: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.540986: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.540667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:26:18.540995: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.540999: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:26:18.541021: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711195: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:26:18.711804: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711806: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711811: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711813: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711811: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711815: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711819: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:26:18.711821: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:41:35.867829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.867849: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.867854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.867868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.867875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.867877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.867891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.867972: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868524: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 15:41:35.868811: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868577: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 15:41:35.868829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868590: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 15:41:35.868835: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868598: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 15:41:35.868846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 15:41:35.868851: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868619: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 15:41:35.868860: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.868697: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: 2023-02-09 15:41:35.868859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.868914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.876228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: 2023-02-09 15:41:35.876210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.876234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.876216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-09 15:41:35.876233: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.876211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-09 15:41:35.876235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.876216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-09 15:41:35.876236: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.876220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-09 15:41:35.876238: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.876247: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876218: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-09 15:41:35.876236: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.876227: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.876244: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-09 15:41:35.876242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.876232: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876234: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:41:35.876252: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 15:41:35.876255: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: 2023-02-09 15:41:35.876256: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 15:41:35.876257: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:41:35.876237: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876238: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 15:41:35.876260: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 15:41:35.876261: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876239: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876242: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 15:41:35.876245: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883765: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883767: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883775: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883776: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883771: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883770: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883771: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:41:35.883788: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883789: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883792: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883794: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883795: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 15:41:35.883795: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.921349: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.921370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.921377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.921389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.921394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.921397: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.921410: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.921408: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923486: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923488: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923490: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923501: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.923501: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.923504: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.923506: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.923512: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.923511: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.923514: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 15:41:35.923582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:41:35.923595: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.056826: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.056854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.056843: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.056841: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.056852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.056854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.056860: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.056884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058576: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058590: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.058590: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.058588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058593: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058609: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.058610: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.058613: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.058613: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.058616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:41:38.058635: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 15:41:38.058639: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 15:41:38.201485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.201502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-09 15:41:38.201630: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.201498: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-09 15:41:38.201627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.201506: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-09 15:41:38.201654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.201507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-09 15:41:38.201658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.201511: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-09 15:41:38.201661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.201510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-09 15:41:38.201671: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.201532: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: 2023-02-09 15:41:38.201677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:41:38.201681: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:41:38.203381: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-09 15:41:38.203322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:41:38.203380: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:41:38.203379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:41:38.203385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-09 15:41:38.203331: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203341: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 15:41:38.203341: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203388: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-09 15:41:38.203342: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 15:41:38.203346: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 15:41:38.203345: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203346: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 15:41:38.203348: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203382: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: 2023-02-09 15:41:38.203371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:41:38.203384: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203381: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:41:38.203392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:41:38.203398: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203398: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203400: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203403: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203404: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203405: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203407: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 15:41:38.203408: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.957597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.957603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.957618: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.957614: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.957613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.957613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.957618: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.957622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959874: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959876: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959876: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:41:39.959887: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.959884: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.959884: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.959889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.959892: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.959892: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.959895: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 15:41:39.959897: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_upper_triang_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +2: Successfully preprocessed all matching files. +2: Successfully preprocessed all matching files. +2: Successfully preprocessed all matching files. +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +6: Building extension module utils... +6: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +6: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +5: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +5: Building extension module utils... +5: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +5: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +3: Loading extension module utils... +1: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +6: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +1: Loading extension module utils... +7: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +7: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +0: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +6: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +6: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils...Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: Loading extension module utils... +2: +2: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +3: +3: Loading extension module utils...Loading extension module utils... +3: +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +7: +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: Loading extension module utils... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/619m22b22b/logs/2817193.out b/619m22b22b/logs/2817193.out new file mode 100644 index 0000000000000000000000000000000000000000..f38e7170e59ec700d706b159bea85d54cabefcf0 --- /dev/null +++ b/619m22b22b/logs/2817193.out @@ -0,0 +1,6693 @@ +Model parameters: d_model 1536 ffw_size 6144 kv_size 128 n_heads 12 n_layers 19 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 19 --hidden-size 1536 --num-attention-heads 12 --kv-channels 128 --ffn-hidden-size 6144 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-619m22b22bval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --no-load-optim --reset-progress --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --tensorboard-dir tensorboard_619m22b22bval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_619m22b22b --load checkpoints_619m22b22b --train-weighted-split-paths-path train4b8.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/2817193.json --zero-stage 0 +START 2817193: Thu 09 Feb 2023 03:08:59 PM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 38.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 43.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 48.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 39.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 56.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 37.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 39.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 46.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 46.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 46.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 43.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 37.0c 80.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 47.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 43.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 48.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 48.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +7: Launching on nid006694 (7/8), master nid006466 port 9999, GPUs 8, CUDA: True +0: Launching on nid006466 (0/8), master nid006466 port 9999, GPUs 8, CUDA: True +1: Launching on nid006467 (1/8), master nid006466 port 9999, GPUs 8, CUDA: True +2: Launching on nid006468 (2/8), master nid006466 port 9999, GPUs 8, CUDA: True +6: Launching on nid006693 (6/8), master nid006466 port 9999, GPUs 8, CUDA: True +5: Launching on nid006691 (5/8), master nid006466 port 9999, GPUs 8, CUDA: True +3: Launching on nid006689 (3/8), master nid006466 port 9999, GPUs 8, CUDA: True +4: Launching on nid006690 (4/8), master nid006466 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/2817193.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... None +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 6144 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1536 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-619m22b22bval +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_619m22b22b +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... None +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 12 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 19 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_619m22b22b +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_619m22b22bval +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-09 17:17:34,209] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +7: > setting tensorboard ... +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.126 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 87 +0: ninja: no work to do. +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 63 +0: [1/1] c++ scaled_masked_softmax_hip.o scaled_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: [1/1] c++ layer_norm_hip_kernel.cuda.o layer_norm_cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so +0: >>> done with compiling and loading fused kernels. Compilation time: 46.410 seconds +0: time to initialize megatron (seconds): 64.250 +0: [after megatron is initialized] datetime: 2023-02-09 17:18:24 +0: building GPT model ... +0: [2023-02-09 17:18:24,411] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-09 17:18:24,412] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-09 17:18:24,412] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 29.89 GB, percent = 5.9% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-09 17:18:26,404] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=26 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: undo +0: 23: MixedFusedLayerNorm +0: 24: EmbeddingPipe +0: 25: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-09 17:18:26,698] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-09 17:18:26,698] [INFO] [utils.py:828:see_memory_usage] MA 1.16 GB Max_MA 1.16 GB CA 1.2 GB Max_CA 1 GB +0: [2023-02-09 17:18:26,698] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 29.94 GB, percent = 5.9% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-09 17:18:26,700] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-09 17:18:41,253] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-09 17:18:41,253] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-09 17:18:41,253] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-09 17:18:41,260] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-09 17:18:41,260] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-09 17:18:41,379] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-09 17:18:41,380] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.17 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-09 17:18:41,380] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.62 GB, percent = 6.1% +6: ninja: no work to do. +6: Time to load utils op: 0.30499887466430664 seconds +6: Time to load utils op: 0.0018532276153564453 seconds +5: ninja: no work to do. +6: Time to load utils op: 0.4214777946472168 seconds +6: Time to load utils op: 0.4226715564727783 seconds +6: Time to load utils op: 0.42708301544189453 seconds +6: Time to load utils op: 0.4283602237701416 seconds +6: Time to load utils op: 0.4297823905944824 seconds +5: Time to load utils op: 0.39386916160583496 seconds +3: Time to load utils op: 0.3724222183227539 seconds +3: Time to load utils op: 0.3719668388366699 seconds +3: Time to load utils op: 0.37242794036865234 seconds +3: Time to load utils op: 0.3638172149658203 seconds +3: Time to load utils op: 0.37198877334594727 secondsTime to load utils op: 0.37197160720825195 secondsTime to load utils op: 0.37242865562438965 seconds +3: +3: +3: Time to load utils op: 0.37313270568847656 seconds +1: Time to load utils op: 0.37718629837036133 seconds +1: Time to load utils op: 0.3777790069580078 seconds +1: Time to load utils op: 0.36882901191711426 seconds +1: Time to load utils op: 0.377183198928833 seconds +1: Time to load utils op: 0.37778687477111816 seconds +1: Time to load utils op: 0.3778102397918701 secondsTime to load utils op: 0.3771946430206299 seconds +1: +1: Time to load utils op: 0.37685561180114746 seconds +7: Time to load utils op: 0.36277246475219727 secondsTime to load utils op: 0.370743989944458 seconds +7: +7: Time to load utils op: 0.37064599990844727 secondsTime to load utils op: 0.37030863761901855 seconds +7: +7: Time to load utils op: 0.37107396125793457 seconds +7: Time to load utils op: 0.37074875831604004 secondsTime to load utils op: 0.3711071014404297 seconds +7: +7: Time to load utils op: 0.3706667423248291 seconds +6: Time to load utils op: 0.001680612564086914 seconds +6: Time to load utils op: 0.0026552677154541016 seconds +2: Time to load utils op: 0.4529550075531006 seconds +2: Time to load utils op: 0.4520576000213623 seconds +2: Time to load utils op: 0.4523429870605469 seconds +2: Time to load utils op: 0.4525148868560791 seconds +2: Time to load utils op: 0.4523918628692627 seconds +2: Time to load utils op: 0.4517943859100342 secondsTime to load utils op: 0.4530937671661377 seconds +2: Time to load utils op: 0.45322084426879883 seconds +2: +6: Time to load utils op: 0.0003337860107421875 seconds +6: Time to load utils op: 0.0021080970764160156 seconds +6: Time to load utils op: 0.0003123283386230469 seconds +5: Time to load utils op: 0.4301412105560303 seconds +5: Time to load utils op: 0.43176960945129395 seconds +5: Time to load utils op: 0.4330451488494873 seconds +5: Time to load utils op: 0.4314568042755127 seconds +5: Time to load utils op: 0.43630504608154297 seconds +5: Time to load utils op: 0.43823885917663574 seconds +5: Time to load utils op: 0.4397876262664795 seconds +0: Time to load utils op: 0.644681453704834 seconds +0: Time to load utils op: 0.5131299495697021 seconds +0: Time to load utils op: 0.5174109935760498 seconds +0: Time to load utils op: 0.5181503295898438 seconds +4: Time to load utils op: 0.46776485443115234 secondsTime to load utils op: 0.47276806831359863 seconds +4: +4: Time to load utils op: 0.472761869430542 seconds +4: Time to load utils op: 0.4727745056152344 secondsTime to load utils op: 0.4725003242492676 seconds +4: +4: Time to load utils op: 0.4728116989135742 secondsTime to load utils op: 0.47280120849609375 secondsTime to load utils op: 0.47278785705566406 seconds +4: +4: +0: Time to load utils op: 0.5193192958831787 seconds +6: Time to load utils op: 0.5131757259368896 seconds +0: Time to load utils op: 0.5201239585876465 seconds +0: Time to load utils op: 0.5212295055389404 seconds +0: Time to load utils op: 0.5230636596679688 seconds +6: Time to load utils op: 0.518409252166748 seconds +6: Time to load utils op: 0.00035262107849121094 seconds +6: Time to load utils op: 0.0003311634063720703 seconds +2: Time to load utils op: 0.009749650955200195 seconds +2: Time to load utils op: 0.014987468719482422 seconds +2: Time to load utils op: 0.016053199768066406 seconds +2: Time to load utils op: 0.015840530395507812 seconds +2: Time to load utils op: 0.014012336730957031 secondsTime to load utils op: 0.016000747680664062 seconds +2: +2: Time to load utils op: 0.014999151229858398 seconds +2: Time to load utils op: 0.015936851501464844 seconds +5: Time to load utils op: 0.0032460689544677734 seconds +5: Time to load utils op: 0.00518345832824707 seconds +0: Time to load utils op: 0.0017523765563964844 seconds +0: Time to load utils op: 0.002321004867553711 seconds +0: [2023-02-09 17:18:42,136] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: Time to load utils op: 0.002912759780883789 seconds +0: Time to load utils op: 0.00289154052734375 seconds +5: Time to load utils op: 0.009171724319458008 seconds +0: [2023-02-09 17:18:42,137] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.15 GB CA 1.22 GB Max_CA 1 GB +0: Time to load utils op: 0.00039577484130859375 seconds +0: Time to load utils op: 0.0023851394653320312 seconds +0: [2023-02-09 17:18:42,137] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.63 GB, percent = 6.1% +0: Time to load utils op: 0.002766132354736328 seconds +3: Time to load utils op: 0.00907444953918457 seconds +5: Time to load utils op: 0.011147737503051758 seconds +1: Time to load utils op: 0.00983285903930664 seconds +7: Time to load utils op: 0.013057708740234375 seconds +3: Time to load utils op: 0.015004634857177734 seconds +3: Time to load utils op: 0.014917373657226562 seconds +3: Time to load utils op: 0.014924287796020508 seconds +3: Time to load utils op: 0.015007495880126953 seconds +3: Time to load utils op: 0.013993501663208008 seconds +3: Time to load utils op: 0.015069723129272461 seconds +3: Time to load utils op: 0.01545572280883789 seconds +7: Time to load utils op: 0.016190767288208008 seconds +7: Time to load utils op: 0.01622152328491211 seconds +7: Time to load utils op: 0.016195297241210938 seconds +7: Time to load utils op: 0.01618361473083496 seconds +7: Time to load utils op: 0.01605987548828125 seconds +7: Time to load utils op: 0.0157928466796875 seconds +1: Time to load utils op: 0.014364480972290039 seconds +1: Time to load utils op: 0.014333724975585938 seconds +1: Time to load utils op: 0.014300823211669922 seconds +1: Time to load utils op: 0.013381719589233398 seconds +1: Time to load utils op: 0.013845682144165039 seconds +1: Time to load utils op: 0.014420509338378906 seconds +1: Time to load utils op: 0.01498866081237793 seconds +7: Time to load utils op: 0.016898632049560547 seconds +5: Time to load utils op: 0.018253564834594727 secondsTime to load utils op: 0.018025636672973633 seconds +5: +5: Time to load utils op: 0.017928361892700195 seconds +5: Time to load utils op: 0.01779317855834961 seconds +4: Time to load utils op: 0.008945226669311523 seconds +4: Time to load utils op: 0.009779691696166992 seconds +4: Time to load utils op: 0.011956214904785156 seconds +4: Time to load utils op: 0.01910400390625 seconds +4: Time to load utils op: 0.019091129302978516 seconds +4: Time to load utils op: 0.018960237503051758 seconds +4: Time to load utils op: 0.01918506622314453 seconds +4: Time to load utils op: 0.019219160079956055 seconds +0: [2023-02-09 17:18:42,269] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-09 17:18:42,269] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 17:18:42,269] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:42,375] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-09 17:18:42,375] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 17:18:42,375] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:42,482] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-09 17:18:42,482] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:42,482] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:42,586] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-09 17:18:42,586] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:42,586] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:42,692] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-09 17:18:42,693] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:42,693] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:42,796] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-09 17:18:42,796] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:42,796] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:42,907] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-09 17:18:42,907] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:42,907] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:43,010] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-09 17:18:43,010] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:43,011] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.77 GB, percent = 6.1% +0: [2023-02-09 17:18:43,011] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-09 17:18:43,011] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-09 17:18:43,011] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-09 17:18:43,011] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-09 17:18:43,011] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-09 17:18:43,012] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-09 17:18:43,013] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-09 17:18:43,014] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.0004169940948486328 seconds +0: [2023-02-09 17:18:43,014] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-09 17:18:43,037] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=26 [0, 26) STAGE_PARAMS=618714624 (618.715M) TOTAL_PARAMS=618714624 (618.715M) UNIQUE_PARAMS=618714624 (618.715M) +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:43,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:43,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:43,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:43,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:43,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:43,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:43,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:43,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:43,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:43,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:43,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:43,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:43,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:43,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:43,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:43,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:43,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:43,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:43,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:43,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:43,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:43,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:43,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:43,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:43,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:43,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:43,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:43,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:43,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:43,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:43,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:43,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:43,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:43,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:43,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:43,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:43,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:43,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:43,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:43,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:43,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:43,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:43,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:43,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:43,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:43,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:43,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:43,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:43,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:43,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:43,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:43,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:43,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:43,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:43,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:43,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:43,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:43,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:43,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:43,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:44,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:44,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:44,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:44,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:44,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:44,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:44,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:44,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:44,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:44,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:44,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:44,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:44,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:44,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:44,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:44,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:44,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:44,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:44,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:44,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:44,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:44,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:44,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:44,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:44,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:44,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:44,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:44,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:44,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:44,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:44,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:44,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:44,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:44,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:44,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:44,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:44,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:44,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:44,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:44,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:44,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:44,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:44,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:44,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:44,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:44,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:44,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:44,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:44,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:44,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:44,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:44,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:44,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:44,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:44,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:44,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:44,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:44,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:44,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:44,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:44,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:44,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:44,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:44,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:44,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:44,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:44,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:44,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:44,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:44,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:44,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:44,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:44,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:44,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:44,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:44,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:44,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:45,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:45,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:45,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:45,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:45,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:45,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:45,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:45,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:45,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:45,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:45,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:45,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:45,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:45,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:45,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:45,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:45,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:45,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:45,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:45,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:45,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:45,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:45,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:45,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:45,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:45,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:45,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:45,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:45,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:45,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:45,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:45,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:45,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:45,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:45,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:45,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:45,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:45,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:45,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:45,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:45,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:45,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:45,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:45,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:45,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:45,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:45,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:45,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:45,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:45,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:45,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:45,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:45,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:45,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:45,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:45,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:45,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:45,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:45,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:45,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:45,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:45,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:45,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:45,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:45,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:45,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:45,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:45,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:45,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:45,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:45,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:45,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:45,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:45,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:45,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:45,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:45,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:45,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:45,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:45,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:45,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:45,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:45,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:45,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:45,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:45,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:45,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:45,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:45,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:45,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:45,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:45,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:45,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:45,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:45,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:45,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:45,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:45,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:45,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:45,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:45,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:45,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:45,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:45,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:45,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:46,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:46,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:46,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:46,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:46,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:46,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:46,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:46,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:46,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:46,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:46,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:46,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:46,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:46,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:46,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:46,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:46,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:46,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:46,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:46,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:46,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:46,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:46,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:46,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:46,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:46,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:46,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:46,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:46,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:46,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:46,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:46,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:46,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:46,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:46,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:46,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:46,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:46,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:46,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:46,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:46,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:46,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:46,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:46,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:46,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:46,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:46,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:46,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:46,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:46,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:46,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:46,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:46,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:46,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:46,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:46,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:46,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:46,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:46,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:46,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:46,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:46,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:46,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:46,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:46,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:46,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:46,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:46,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:46,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:46,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:46,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:46,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:46,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:46,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:46,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:46,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:46,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:46,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:46,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:46,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:46,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:46,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:46,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:46,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:46,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:46,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:46,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:46,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:46,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:46,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:46,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:46,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:46,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:46,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:46,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:46,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:47,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:47,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:47,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:47,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:47,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:47,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:47,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:47,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:47,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:47,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:47,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:47,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:47,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:47,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:47,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:47,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:47,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:47,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:47,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:47,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:47,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:47,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:47,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:47,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:47,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:47,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:47,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:47,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:47,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:47,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:47,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:47,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:47,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:47,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:47,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:47,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:47,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:47,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:47,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:47,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:47,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:47,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:47,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:47,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:47,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:47,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:47,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:47,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:47,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:47,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:47,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:47,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:47,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:47,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:47,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:47,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:47,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:47,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:47,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:47,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:47,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:47,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:47,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:47,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:47,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:47,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:47,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:47,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:47,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:47,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:47,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:47,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:47,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:47,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:47,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:47,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:47,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:47,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:48,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:48,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:48,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:48,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:48,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:48,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:48,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:48,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:48,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:48,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:48,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:48,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:48,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:48,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:48,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:48,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:48,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:48,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:48,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:48,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:48,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:48,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:48,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:48,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:48,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:48,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:48,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:48,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:48,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:48,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:48,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:48,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +0: [2023-02-09 17:18:48,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:48,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:48,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:48,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:48,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:48,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:48,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:48,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:48,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:48,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:48,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:48,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:48,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:48,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:48,598] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +3: [2023-02-09 17:18:48,602] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +7: [2023-02-09 17:18:48,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,603] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +7: [2023-02-09 17:18:48,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +0: [2023-02-09 17:18:48,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,613] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +0: [2023-02-09 17:18:48,617] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +0: [2023-02-09 17:18:48,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,619] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +0: [2023-02-09 17:18:48,622] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +0: [2023-02-09 17:18:48,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,633] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +4: [2023-02-09 17:18:48,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,635] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +0: [2023-02-09 17:18:48,638] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +4: [2023-02-09 17:18:48,639] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +2: [2023-02-09 17:18:48,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,647] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +2: [2023-02-09 17:18:48,651] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +7: [2023-02-09 17:18:48,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,660] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +1: [2023-02-09 17:18:48,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:48,662] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +7: [2023-02-09 17:18:48,664] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +6: [2023-02-09 17:18:48,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:48,665] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +1: [2023-02-09 17:18:48,666] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +6: [2023-02-09 17:18:48,669] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +7: [2023-02-09 17:18:48,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,671] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +7: [2023-02-09 17:18:48,675] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +4: [2023-02-09 17:18:48,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,680] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +4: [2023-02-09 17:18:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,681] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +4: [2023-02-09 17:18:48,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,682] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +4: [2023-02-09 17:18:48,685] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +4: [2023-02-09 17:18:48,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,685] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +4: [2023-02-09 17:18:48,686] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +4: [2023-02-09 17:18:48,686] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +0: [2023-02-09 17:18:48,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,688] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +4: [2023-02-09 17:18:48,690] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +0: [2023-02-09 17:18:48,692] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +7: [2023-02-09 17:18:48,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,706] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +2: [2023-02-09 17:18:48,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,709] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +2: [2023-02-09 17:18:48,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,711] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +2: [2023-02-09 17:18:48,711] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +2: [2023-02-09 17:18:48,715] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +2: [2023-02-09 17:18:48,716] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +4: [2023-02-09 17:18:48,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,727] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +4: [2023-02-09 17:18:48,732] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +5: [2023-02-09 17:18:48,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:48,741] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +5: [2023-02-09 17:18:48,747] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +1: [2023-02-09 17:18:48,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:48,753] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +1: [2023-02-09 17:18:48,757] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +7: [2023-02-09 17:18:48,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,758] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +7: [2023-02-09 17:18:48,763] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +6: [2023-02-09 17:18:48,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:48,765] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +6: [2023-02-09 17:18:48,769] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +5: [2023-02-09 17:18:48,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:48,768] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +5: [2023-02-09 17:18:48,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:48,769] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +5: [2023-02-09 17:18:48,772] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +5: [2023-02-09 17:18:48,773] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +2: [2023-02-09 17:18:48,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,774] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +2: [2023-02-09 17:18:48,778] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +5: [2023-02-09 17:18:48,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:48,787] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +5: [2023-02-09 17:18:48,791] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +6: [2023-02-09 17:18:48,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:48,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +6: [2023-02-09 17:18:48,802] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +0: [2023-02-09 17:18:48,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,803] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +0: [2023-02-09 17:18:48,807] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +3: [2023-02-09 17:18:48,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:48,824] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +3: [2023-02-09 17:18:48,828] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +2: [2023-02-09 17:18:48,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,829] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +2: [2023-02-09 17:18:48,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,830] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +2: [2023-02-09 17:18:48,833] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +2: [2023-02-09 17:18:48,834] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +3: [2023-02-09 17:18:48,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:48,841] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +3: [2023-02-09 17:18:48,845] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +6: [2023-02-09 17:18:48,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:48,849] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +6: [2023-02-09 17:18:48,853] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +1: [2023-02-09 17:18:48,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:48,863] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +1: [2023-02-09 17:18:48,868] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +2: [2023-02-09 17:18:48,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,875] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +2: [2023-02-09 17:18:48,881] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +6: [2023-02-09 17:18:48,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:48,890] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +4: [2023-02-09 17:18:48,890] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +4: [2023-02-09 17:18:48,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:48,892] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +6: [2023-02-09 17:18:48,894] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +4: [2023-02-09 17:18:48,894] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +2: [2023-02-09 17:18:48,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,895] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +4: [2023-02-09 17:18:48,896] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +1: [2023-02-09 17:18:48,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:48,898] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +5: [2023-02-09 17:18:48,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:48,899] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +5: [2023-02-09 17:18:48,899] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +0: [2023-02-09 17:18:48,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,900] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +7: [2023-02-09 17:18:48,900] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +1: [2023-02-09 17:18:48,902] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +0: [2023-02-09 17:18:48,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,903] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +5: [2023-02-09 17:18:48,903] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +7: [2023-02-09 17:18:48,904] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +0: [2023-02-09 17:18:48,904] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +6: [2023-02-09 17:18:48,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:48,906] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +0: [2023-02-09 17:18:48,908] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +6: [2023-02-09 17:18:48,910] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +1: [2023-02-09 17:18:48,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:48,914] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +1: [2023-02-09 17:18:48,917] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +3: [2023-02-09 17:18:48,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:48,932] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +3: [2023-02-09 17:18:48,936] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +6: [2023-02-09 17:18:48,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:48,939] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +3: [2023-02-09 17:18:48,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:48,941] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +6: [2023-02-09 17:18:48,944] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +3: [2023-02-09 17:18:48,945] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +1: [2023-02-09 17:18:48,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:48,947] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +1: [2023-02-09 17:18:48,951] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +7: [2023-02-09 17:18:48,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:48,952] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +7: [2023-02-09 17:18:48,957] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +5: [2023-02-09 17:18:48,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:48,957] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +5: [2023-02-09 17:18:48,961] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +0: [2023-02-09 17:18:48,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:48,974] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +0: [2023-02-09 17:18:48,978] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +3: [2023-02-09 17:18:48,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:48,983] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +3: [2023-02-09 17:18:48,987] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +1: [2023-02-09 17:18:48,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:48,988] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +1: [2023-02-09 17:18:48,993] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +3: [2023-02-09 17:18:49,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:49,009] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +3: [2023-02-09 17:18:49,013] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +7: [2023-02-09 17:18:49,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:49,021] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +7: [2023-02-09 17:18:49,025] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +1: [2023-02-09 17:18:49,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:49,057] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +1: [2023-02-09 17:18:49,062] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +6: [2023-02-09 17:18:49,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:49,219] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +6: [2023-02-09 17:18:49,223] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +3: [2023-02-09 17:18:49,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:49,298] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +3: [2023-02-09 17:18:49,302] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +5: [2023-02-09 17:18:49,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:49,962] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +5: [2023-02-09 17:18:49,966] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +5: [2023-02-09 17:18:50,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b22b/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:50,032] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +5: [2023-02-09 17:18:50,036] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +0: successfully loaded checkpoint from checkpoints_619m22b22b at iteration 0 +7: time (ms) | load-checkpoint: 7007.11 +0: estimated model parameters: 0.618714624 +0: estimated model parameters without embeddings: 0.538301952 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-09 17:18:51 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.033515 seconds +0: number of documents: 10028711 +0: > dataset split: +0: train: +0: document indices in [0, 10028711) total of 10028711 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.046 seconds +0: total number of samples: 2342263 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.024508 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.010 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-09 17:19:05 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 26153.35 | train/valid/test-data-iterators-setup: 14333.75 +0: [after training is done] datetime: 2023-02-09 17:19:05 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 2.900172E+00 | lm loss PPL: 1.817728E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 2817193: Thu 09 Feb 2023 05:19:37 PM EET diff --git a/619m22b22b/sbatch_619m22b22b.sh b/619m22b22b/sbatch_619m22b22b.sh new file mode 100755 index 0000000000000000000000000000000000000000..6c119b02e641438cf9dffa62ea14c9ee0f9805d8 --- /dev/null +++ b/619m22b22b/sbatch_619m22b22b.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m22b22b + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train22b.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_22B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 21500000000 +# -> Samples: 10498047 +TRAIN_SAMPLES=10_498_047 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 104_980 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m22b22b/sbatch_619m22b22bval.sh b/619m22b22b/sbatch_619m22b22bval.sh new file mode 100644 index 0000000000000000000000000000000000000000..98ce805a9a3fa414b55713e20089d9429af56aab --- /dev/null +++ b/619m22b22b/sbatch_619m22b22bval.sh @@ -0,0 +1,167 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m22b22bval +VARIANT_CKPT=619m22b22b + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train4b8.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 21500000000 +# -> Samples: 10498047 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --override-lr-scheduler \ + --no-load-optim \ + --reset-progress \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675507284.nid005541.111295.0 b/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675507284.nid005541.111295.0 new file mode 100644 index 0000000000000000000000000000000000000000..02345e0775016a15364813e93e9d0af5bae1f909 --- /dev/null +++ b/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675507284.nid005541.111295.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f0da8343b2be3873f465c19d3854e86beee72f9f39cb042e644b875b6f59b4e +size 40 diff --git a/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675539324.nid006152.105943.0 b/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675539324.nid006152.105943.0 new file mode 100644 index 0000000000000000000000000000000000000000..5183149e0fa901621af9ae9aefe23d88c70b047b --- /dev/null +++ b/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675539324.nid006152.105943.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75efe0e7cc5b1b5627ff5cb33fb936ac2ed424cbdae76ed3ca3e752e8deec639 +size 49131476 diff --git a/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675874501.nid006485.99075.0 b/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675874501.nid006485.99075.0 new file mode 100644 index 0000000000000000000000000000000000000000..2d17209f71c83a03e8ab350546b905acbfe72bef --- /dev/null +++ b/619m22b22b/tensorboard_619m22b22b/events.out.tfevents.1675874501.nid006485.99075.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8a51aba835f7dba7837cc1da6453a52e1a2ec86020844611e960d9c396908d7 +size 25121238 diff --git a/619m22b22b/tensorboard_619m22b22bval/events.out.tfevents.1675955854.nid006694.100350.0 b/619m22b22b/tensorboard_619m22b22bval/events.out.tfevents.1675955854.nid006694.100350.0 new file mode 100644 index 0000000000000000000000000000000000000000..ad30041152d0fae3da9a3d5349adc7bd8ed25796 --- /dev/null +++ b/619m22b22b/tensorboard_619m22b22bval/events.out.tfevents.1675955854.nid006694.100350.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefe4fcb16ee43927c69f74dbb181aeac17e9970be5ff06f3a384f9fc46d26c1 +size 980 diff --git a/619m22b22b/transformers/config.json b/619m22b22b/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a1592eaab83db706537cb6afd4da42ec9828de43 --- /dev/null +++ b/619m22b22b/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50304, "n_positions": 2048, "n_embd": 1536, "n_layer": 19, "n_head": 12, "n_inner": 6144, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/619m22b22b/transformers/pytorch_model.bin b/619m22b22b/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8e94cbd5e62028da1ad5eb20f297aee553daf18e --- /dev/null +++ b/619m22b22b/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a80a67f7bf90f14af2b66e4d35d7c0754481b7ba9d42be0aac82a76a8ae8341b +size 1396896901 diff --git a/619m22b400m/tensorboard_619m22b400mval/events.out.tfevents.1678950236.nid007044.107179.0 b/619m22b400m/tensorboard_619m22b400mval/events.out.tfevents.1678950236.nid007044.107179.0 new file mode 100644 index 0000000000000000000000000000000000000000..73ddc554e691d72ca1d8f4f246bcad5e88b45e3f --- /dev/null +++ b/619m22b400m/tensorboard_619m22b400mval/events.out.tfevents.1678950236.nid007044.107179.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9995ad2bdd9c11bdce30349246db46152c813a1159aa30c03e4e94e90ca11a1 +size 980 diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..489370f93336806bb9b0d972f85efc4cb6d0081e --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.3099668612165794, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03007866592593964}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06460958235496345, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017809066736377858}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.25749913361086507, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004704092035936928}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09558920039471365, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002149036988481146}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.031187491849803337, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011899008435486756}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.12656441757990483, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00313676095690092}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04582551071096432, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013261745370059053}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.06178350535067078, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016362696987831417}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2508928298018324, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004573446137820672}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09186630288423041, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019745485370318864}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06149602815890292, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016810240234989833}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.24666611039273245, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004421778464668269}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09105569938711074, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002004179378466674}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..0a06b91cd0c88659e84c0937710cb5c0e1cd10f3 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.29130877143455586, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0241959153769584}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.061208429338304574, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014402546745354069}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.28859748128262924, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0049986158310419354}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09445919469734558, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001936296232548762}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.028263099280238797, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008784314903876122}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13604193300293232, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0032346419377748733}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04361066921501477, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001199759977794257}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05834596522178436, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012947018575173474}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.27784325004495686, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0047400498630427775}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09036600743610688, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017702218609310719}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05847161583671669, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013503259579416152}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.27532137412331964, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00463181262519618}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09022680216928644, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018145691342534367}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..46cfca6ec5389b171697e63b5c56ac37bb49ce4d --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.26676093352000974, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01745767099704443}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05941054322249251, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013961426249521755}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2835970082022701, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004788224891673027}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09217510177969336, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018831049937955784}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.027316816074161285, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008486872680257794}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.1354275420032431, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003183687122052663}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.042424120797488585, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011719244443792272}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05626626520925391, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001242253342941174}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.27095597028481616, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004453368974145521}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.0876120055713282, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017024934701633891}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05698004937770005, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013218209896847077}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.27207800066344545, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004514833883372158}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.08841779444595972, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017834242097373026}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..f59a27f9d027b9aafe70a4b7aad0989800a1a4ac --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.28612944934937573, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.030058507463981785}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05858674457588962, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001331757166815278}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.27805772725490513, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004621635420794643}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09107726735341869, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018176734285625148}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02669592353017647, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008008577341120715}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13073693050689278, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0030684516936530922}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.041642636541249896, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011260991098264973}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.0556444229114036, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011944680158197211}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2654099060152725, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004278095570672299}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.08669913456777828, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016464359763810076}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05629907335785418, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012605456096568002}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.26728724980476815, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004336618966824677}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.08753387249247915, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017181361371174542}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..8d9e74cba397837b9950079d99c8addb8b89aa42 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.26705903217674426, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02405825312974743}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06086053476185062, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014318320841098697}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.28542751510426556, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004692910204968811}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09372278299065458, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018460404864548486}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.027485650453758618, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008756611121258637}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.13419537296922995, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003122488099367849}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.04240340414142105, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011357663748880447}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.057676657207041945, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012931118119002293}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2722634215290626, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00437854396102679}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.08903710986289586, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016750629187636654}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.058444776332263366, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013576642072023544}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.27420324540126445, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0044098612417007715}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09001878404549397, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017466945253972782}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..7e10ade7d8907eced67d3e2a24b8d78d90b4bd14 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.32955887349530977, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.028674503871665574}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06531639829304567, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018118510618833096}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.2925041739048238, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004813556224471976}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.09759507774790617, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019916427117499195}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.030671504391052696, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011385946802848516}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.14139940825924777, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0032645688758915227}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.045510066340999515, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012521676735104974}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.061620792950991723, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016331920676026356}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.2780347555856145, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004448204648030782}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.09234102161235082, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001781843163970405}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.06243557617018187, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016873068321191943}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.2804738669361521, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004518723891360148}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.09340724567530662, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018560344045322032}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..cb38cd5923f9cce23f83271d18d4773bec85acfd --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.12608148311182882, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001877790179736341}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.2049303749096855, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0025880256283742705}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.14417929178250435, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001842074002833332}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.021892949427677305, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006292392217955498}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.037328795648641035, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001109797618553952}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.025455042866036937, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007020843746137193}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.10138125064278208, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013701882816016511}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.17034733773499464, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020889089694998476}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.1172503924682141, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013592087451697002}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.11579468211486772, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017076861625592678}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.18934638686212538, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002385952331190746}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.13269967531471152, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016802961690273025}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.0659937881497992, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05080391562111616}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..456b2c3d4825112ebbc867aa9dd3da806232d328 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.13086599178590658, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001672077706598485}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.20720602482820027, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024533756253109274}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.146875208754622, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016188663245510955}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.01792165073770399, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006428200847183632}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.03061200720033379, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011202861598998129}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.02035969927408505, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006205740177121067}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09572436493015317, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011588355930183143}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.1552970781871763, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018502784378049198}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.10777539236344279, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010799004074572}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.12289889278902484, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015656539473863284}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.19479380254592224, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00229544496822277}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.13782959522383184, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015003752806397137}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.9957790449656027, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02530207633623386}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..3295c17546da974e2811b55659f4c43f867d1030 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.1283280211841892, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018471880476730181}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.19588255520964618, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024808008313328897}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.14029812169934233, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016746773304471378}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.018646070313095908, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006575760758617186}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.030141933661931047, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010922246627428195}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.02068240392527567, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006586430976373696}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09962500896923289, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013578472583766657}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.156123382155486, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001955944209275228}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.10962969522414868, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012104753879873735}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.11997014021044887, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017299508610180456}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.18313685603265942, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023053627613329746}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.1309842621150673, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015447855054700223}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.0261550894771894, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03538435325757124}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4af06869c433b10bacfdde3bc9491a251349cafb --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.1174937613247287, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002215169887824098}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.16673838559605367, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002632888074263044}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.12147663727978152, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018368431033577758}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.019684843075251483, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008735431083143301}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.028644261873361904, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010691604436933462}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.020106815196566395, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006821011446245043}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09329668669102507, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017731591089089934}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.13511709856165047, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021379505707010355}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.09666295792962627, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014039308878508574}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.10937771197761693, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002069130170725252}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.15562267064023425, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002460546591552193}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.11304180944340365, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001697758818846021}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 1.2439413140181725, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08021377052333785}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..207910ead986c53fc57a77bc23e16c01e830ddd1 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.04156484323618865, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017463462545007212}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.05834009675533347, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002218925403012533}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.04098065480515757, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014835204076038898}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.00736474207758501, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006168003872532538}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.010833168232581908, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007793950988884573}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.007236113220479305, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00045629416756844566}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.034147189968935676, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014824990274979345}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.0480868680398638, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018328691284379799}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.033279909424772915, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011802065589786679}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.03891404857872913, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001653718902747482}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.054089485989470866, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020505430260237653}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.03810821262468117, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001380639279965432}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.2661907581558446, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03202842695354204}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..a26b2e6e741d72c5530fb387530bd10d86ebf46a --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.007397940774007664, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007992853925777316}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.010120982205394452, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0010166711864938761}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.007348718460293708, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0007282980782455469}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.00128680284734034, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00022089770706856912}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0018727716849492996, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00035993430172752866}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.001289849014264577, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00020396030875762924}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.006134073610949074, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.000671334380732274}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.00856899865908427, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0008562869548585046}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0060987117278543486, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005965491826266616}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.0069649726845215495, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.000761180679286655}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.009488299018839004, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0009502861242391249}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.006898026781657591, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006850161593107851}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 4.1838551572050286e-07, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 8.539089327004177e-07}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..eb08388b09e2186265afd2361270ca4e515b4c53 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 0.3337118408961487, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05202728879470043}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.06853867384946524, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012188313815522228}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.1265014794105398, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002527874019892803}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.0817116149040806, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00153613382148468}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.011247369789676413, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00033868159563215023}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.025934933456238086, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007755385845050604}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.015367923182043539, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00045601328888639393}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.06731071739815976, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011879498508703882}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.1256906248949207, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025226724722029028}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.08081655857040615, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015247741746541353}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.05420319385768004, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0009667468979072273}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.09524325606060634, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001900471716812503}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.06255171257305257, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0011543628902571132}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..04e61c144c72a1f8981fdfa3d7279c54d447c8ac --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 8.580785815319153, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12050962641485825}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.43028883128238876, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0027777109499330867}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.3879649870783398, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002764878871199931}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.38545334170928763, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002112571759353758}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.17625411435748609, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002134450890393921}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.15821064491984946, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018746791495472526}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.15599769101716504, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0016286782168266891}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.30818091438114104, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0023544378053750622}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.27542498756422007, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002136202440055506}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2737515668591034, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016582477249727728}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.35631742031115066, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0026424737299773063}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3201295543710165, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002504264887357116}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3182037827678809, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002003091072340103}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4b2ea823d3183d05e221b1d05bf4cc3d822d0f5c --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 10.179948100725445, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.15921732874874983}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4701246692346356, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002973249692601118}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.40133010839486466, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002693499242079775}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.41061062362310147, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021332461778007916}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.21034647711506468, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002262895180766459}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.17752340440327224, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0019231348008012746}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.18123862274918498, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017413899623482422}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3398832332009184, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002529788139802345}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.28899486933645147, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002210430142434923}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2956205579339057, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00182234637449327}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.3854788394930683, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002765196187394664}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3281945242802255, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024573282174066477}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.33602585264600926, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020441813309374983}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..5d6ff228308385b28da0e1bb84fd97e71c47a362 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 10.789731844190179, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12017045252862463}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4829832996859665, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0030484074957686427}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4018418348119394, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026280744814466636}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4160769563473889, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021413655848613504}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.22202793412057192, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0023082250433674803}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.18324630983014636, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00194920314491893}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.18933351465139317, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017958855840332813}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.35067227778777876, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002546479986477843}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.2916905944938416, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002177559260296222}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.30161525481953744, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018405518808223482}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.399851493008314, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0028835232605792824}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.331718034201341, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002428440395770757}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3437678815217894, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002090902101997608}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..f320f86560a825b736be450642a309654588d583 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.195536151651975, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.14805710268987346}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4870389029505203, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0030060268449702385}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.4067622516175034, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002601979181985486}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.4214487218578573, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002125125404165521}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.22789485913434965, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0022923772972932004}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.1884880182238668, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0019387359058115024}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.19498627841697833, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017901321774556802}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.3527544385920249, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0025426680941678494}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.2950536921267289, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022120407027767914}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3049842055805278, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018626579493065487}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.40283848434682135, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0028488718557740248}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3365908664066991, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024526373176337896}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.34847134730498985, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002103441240841663}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..c4ff5dc3ac73c745b5a1e3ef74edbe775691517d --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 11.343200572605854, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11916265276724415}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.4910389116851647, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003058654910112927}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.41088969881696075, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0025168338728330086}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.42600477331701564, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002088618061078895}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.23107977486062162, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0023153275909847428}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.1911427232037733, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001892959619131353}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.1979775765065611, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0017651327620435838}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.35638430160338863, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002569303538249397}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.29912750443223085, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021726065521660725}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.3091536495871777, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018398480793350466}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.4072928239657206, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0028945881851336897}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.3403562650262804, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023737553325940625}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.3528767656515376, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020652380895958976}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_0.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0eb073ef726c87f0ffb624ff06be16d8534c69ab --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.13104806885016143, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016696061533377064}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.32207440240008434, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003806583731509898}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.18408359580247777, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022265545774168786}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.023385645323327594, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008282879766144026}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.05994961316777745, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0021516926902610404}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.033224548947791005, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011698700374986966}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0998987469408451, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001181472858665789}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.2480871740164947, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0028872095133784833}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.14066905424197854, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001589869491949137}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.1022129966770997, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013625920887856532}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.2535042559665434, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003307409271743902}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.14392005845390982, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001850757026152593}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.1792443767397391, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0820008206453002}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_1.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ffdd2a979ab07a5d93e4b32a9fce59826defca07 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.10336243780351441, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015685126380048779}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2585747998651502, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003590429754190019}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.14584396339276617, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002102637664735149}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.014720209928842373, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000681021361327272}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03777011619310744, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017739592882698512}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.020919317079379954, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009622499157490324}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0821069008768099, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011428484900625185}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.20740543017081056, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0027830199070734097}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11612700909117789, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001545382459053668}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.08210160844811192, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012398914712683133}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.20732540894587975, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002979668668234706}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.1161173925093962, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016784793729300376}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.7333669023744487, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13000332641058426}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_2.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..2bec9263313f1b971a8436007f6ef8b1161cd8d7 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.10294913437204971, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001493869061603878}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.2593099601355609, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003449879633572733}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.1456451387246689, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002008514692784554}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.013497535910280573, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006453882484652132}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03494343970894335, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017020373666591004}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.019238022729627213, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009148428605337303}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.08180941551059284, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011179690230981208}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.20760572143774833, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002691010842371602}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11593275835550473, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015093999333849243}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.08204189444833064, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011860547696614754}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.20809616887189838, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002858371023942416}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.11626396730638126, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016082221026397108}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6768758898757194, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.053385621625246696}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_3.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..078868cf1d4963dead2bdbf68a83c4f647933291 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09927106085703968, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016343150158814772}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.24287641032731458, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037267800062189636}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.13781067831884908, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021174515290862607}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.012812200577788478, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006581387038423858}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03307527286460339, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001751247026619044}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.018121988245233207, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009274218042565987}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07901686664376605, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001249903353276574}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.19452412633663918, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002939914567539106}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10987036544353425, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016232442934585429}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.07967747223853743, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013083430563035213}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.19619096905710873, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003055668058044831}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.11072488889531576, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016889944652790365}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6820213665560544, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05333356486252003}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_4.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..1fc59811cc85d45a8686b876f48f230267b6b424 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.03251077542517423, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00213991755859698}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.05897386947594356, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0033433004286718858}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.036965910785755694, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020433261035716077}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0032640836548389874, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004175254160021555}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.007119053240914698, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008708108424698816}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.004171377402417491, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00048514268965227494}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.02634109151345856, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0018307262109688949}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.04715698364480056, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026789790488207796}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.02943448947716526, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016172466516820967}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.027004915852553733, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001864765148099242}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.04844961929509366, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027766936705316205}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.03027547015903745, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016766053945180429}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.30594617026264165, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0676970728415138}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_5.json b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..0178cd80e73b4297b1a6faaca5600bb55cff63d5 --- /dev/null +++ b/619m22b4b8/evaluation/generation/agg.619m22b4b8_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1 @@ +{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0026814100964168137, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007831104236380066}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0022093761015055374, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0006066899354878547}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0023769191008089527, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.000664374120380284}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0005020375788966244, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00027564515577918684}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0003480638857997348, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000171178861038322}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0004044229735136974, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00020796051307837393}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.002048403417496238, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006011286336239785}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0016722359345367028, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00045762893541433525}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0018113358252006796, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005084451143738701}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.002167331876280997, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.000612910003152478}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0018286673479048533, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0004992679324315411}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0019434641482280037, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.000531162691535884}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 8.851716353109152e-37, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 9.796533431961898e-31}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ec72436ddd7e603a8131a05f6ab467f912c9baf8 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c75c8ba5b4ca39c899b0e0160c44cc682edc66e2a6f1ccbc87bf229998e536d0 +size 4158985 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f9e0bbf94e062f7bdddbf058b2c06301bd3a0806 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c34207507626869e6994822f3d6995fe80c0f77b3d376a2c3fe7cee4bc7244f1 +size 5088471 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b065e7493a9c9c4e966a0b2d696105b7a1028282 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a5d53d9561c9dc55d8a69721f80c113fda41313c0c054b2fca53b692e1c5d7e +size 5991107 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..72f30f59f53ca910871c929016687c0f110948f6 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61afc4cc460834b5c839e10b90dae307ff2a9e7ae6642d70c88ac868cfd85723 +size 6905711 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e5870b5ecd53ab5b7599c8beee4967dec8a22ba7 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d59bdc61bca96972ff2398ccb4268b594bc07871f6faa0641a5c558c543750c +size 7806844 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..dae023e7bcb8ff1943480525735231b2499577b6 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2ad0b1ad06c1d84a340ecbedf135b91ca73f9f6cd680a91bbc158a254c4a8ff +size 8697501 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..78140970761da3bf6148cb8c017bbc95f3915be7 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee1beabb7f03f31e67b1e0f342a6f50cdb4690a41c00ce793d9d28c627e9e375 +size 7655358 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4ec333b91d729501117889d3ff4165194c5e0349 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2f8afb95897e8934da6176834e3968692e2149ad4f0932a1855220e488bbaa +size 13308273 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..764b7ac9e93af0f8d19fa6ab4a82601f079fc91b --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd5e0299831a097e878ad43eb9aac732e2affa63b31a50ea907f96a35c1483e +size 18884517 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f42e1491412f39cd3bea69104ead701710bfbb20 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d3500926103c28329b5fe3016bd81110ecf3ccfda382f264f8ddcdfd988385e +size 24302207 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..dc0d9668a24b67d9e41dc717d3d27b6b08d6104e --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bce45d1a4dbd4aee3f4c0e399741ccb90e1d33882462d6b56277905232e4e45c +size 29460787 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cfe231512f75e813414fe33803c4e18f654c3508 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b025e0f34e09943aadf0b61cef77d2b13cedff7c25d404290104eac75340057 +size 34798292 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..bdbca9bff21956da3b443b0251fc34a4a3d99018 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:092e5af42217ecccd9cf1f602cc71d25574df247bcfb1dadd236d2312caf4ea8 +size 4058832 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..556295472d71e589cfc8464834d2a2a38b1dd1d4 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:199266c12eea454a156b8f1ee5c9cb59d364ddb6a0bd43d4650b7d6e0141a6e1 +size 5076586 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f597ec1e2c3e0ffbc42942c8b2b31395c86bc7b5 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76d6f68ea3dd95e18c8f12675ce63eda5f76a1d12a2ac8aad82f0d49612dc889 +size 6144877 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f7af894e48737dcc6ac1d489b31c24c3d7e3b5bf --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f8183589041f34ab8086d295904c9e31684c85b990a5ee9c1beda7cadfa1be7 +size 7221364 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0ba46cc036ac76eed50df3503e6de9d5c08991fb --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8091afd55837dffe9c01fa6409c6a218cb68e25c43e1756849ac9727eeb2730 +size 8300683 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a9e31e63b945c056f6d5f4e4c838fdfd62112a7a --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:addfee276d3bf7b1687a4514947e176f5b0b188a2cb4a23b95ee7e6236b391bb +size 9391863 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_0.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_0.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4e7e0061066d0a30f32e2078199805a30d22a6f0 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_0.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f98ea306a4d73d4366e53db7a1317c77e43e27f227da91a16a4bdfb6c70c7ad9 +size 2838849 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_1.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_1.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0e4199dcadc1026f984df16163b8c78a486d0f44 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_1.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19af961df3def452aed83a9b900b9753c5bd56478be1f054d9c0704653688e28 +size 5106640 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_2.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_2.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5ee60facb890ac8e5333b00a1ffb43736a6096ec --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_2.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89c833abdf51962ab27376c80297447317a6873e50935e3d0f2f26a6e316845c +size 7380031 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_3.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_3.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..aeefbcee737418a11f220427fcbc317f9d3fb92c --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_3.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:643db98364effe954fb47a28b50abcd889cd1ce193999b1cc6cf1f41be28e1e8 +size 9650319 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_4.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_4.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ac5244e4d7c0211d9aaaeda390cdc55ce1ac44f1 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_4.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7168d8d0cbed4c1a97032d1c302513b9a7e6290a61f32ef54daefec5f2caea1 +size 11674036 diff --git a/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_5.jsonl b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_5.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..45d0fd239c86645975b21521091f40691c049d92 --- /dev/null +++ b/619m22b4b8/evaluation/generation/examples.619m22b4b8_gem_xsum_article_DOC_summary_5.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8db3356823f95547825df6becd340ad2a6aebe7b929c7a8fbf7a56cbbd8e98ee +size 13897502 diff --git a/619m22b4b8/evaluation/generation/merged.csv b/619m22b4b8/evaluation/generation/merged.csv new file mode 100644 index 0000000000000000000000000000000000000000..d652ae8f951224c68bcbbaeaaefed250505696af --- /dev/null +++ b/619m22b4b8/evaluation/generation/merged.csv @@ -0,0 +1,53 @@ +dataset,fewshots,prompt,metric,value +e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.015367923182043539 +e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.015367923182043539 +e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.15599769101716504 +e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.15599769101716504 +e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.18123862274918498 +e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.18123862274918498 +e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.18933351465139317 +e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.18933351465139317 +e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.19498627841697833 +e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.19498627841697833 +e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.1979775765065611 +e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.1979775765065611 +e2e_nlg_cleaned,5,average,multiple,0.15581693442055436 +gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.033224548947791005 +gem_xsum,0,median,rouge2_fmeasure,0.033224548947791005 +gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.020919317079379954 +gem_xsum,1,median,rouge2_fmeasure,0.020919317079379954 +gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.019238022729627213 +gem_xsum,2,median,rouge2_fmeasure,0.019238022729627213 +gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.018121988245233207 +gem_xsum,3,median,rouge2_fmeasure,0.018121988245233207 +gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.004171377402417491 +gem_xsum,4,median,rouge2_fmeasure,0.004171377402417491 +gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.0004044229735136974 +gem_xsum,5,median,rouge2_fmeasure,0.0004044229735136974 +gem_xsum,5,average,multiple,0.01601327956299376 +web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.04582551071096432 +web_nlg_en,0,median,rouge2_fmeasure,0.04582551071096432 +web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.04361066921501477 +web_nlg_en,1,median,rouge2_fmeasure,0.04361066921501477 +web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.042424120797488585 +web_nlg_en,2,median,rouge2_fmeasure,0.042424120797488585 +web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.041642636541249896 +web_nlg_en,3,median,rouge2_fmeasure,0.041642636541249896 +web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.04240340414142105 +web_nlg_en,4,median,rouge2_fmeasure,0.04240340414142105 +web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.045510066340999515 +web_nlg_en,5,median,rouge2_fmeasure,0.045510066340999515 +web_nlg_en,5,average,multiple,0.04356940129118969 +wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.025455042866036937 +wiki_lingua_en,0,median,rouge2_fmeasure,0.025455042866036937 +wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.02035969927408505 +wiki_lingua_en,1,median,rouge2_fmeasure,0.02035969927408505 +wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.02068240392527567 +wiki_lingua_en,2,median,rouge2_fmeasure,0.02068240392527567 +wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.020106815196566395 +wiki_lingua_en,3,median,rouge2_fmeasure,0.020106815196566395 +wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.007236113220479305 +wiki_lingua_en,4,median,rouge2_fmeasure,0.007236113220479305 +wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.001289849014264577 +wiki_lingua_en,5,median,rouge2_fmeasure,0.001289849014264577 +wiki_lingua_en,5,average,multiple,0.01585498724945132 diff --git a/619m22b4b8/evaluation/generation/merged.json b/619m22b4b8/evaluation/generation/merged.json new file mode 100644 index 0000000000000000000000000000000000000000..0ee9d0b6e752e971f798989375d3fd4ab343bed8 --- /dev/null +++ b/619m22b4b8/evaluation/generation/merged.json @@ -0,0 +1 @@ +{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.3099668612165794, "bleu_stderr": 0.03007866592593964, "rouge1_fmeasure": 0.09558920039471365, "rouge1_fmeasure_stderr": 0.002149036988481146, "rouge1_precision": 0.06460958235496345, "rouge1_precision_stderr": 0.0017809066736377858, "rouge1_recall": 0.25749913361086507, "rouge1_recall_stderr": 0.004704092035936928, "rouge2_fmeasure": 0.04582551071096432, "rouge2_fmeasure_stderr": 0.0013261745370059053, "rouge2_precision": 0.031187491849803337, "rouge2_precision_stderr": 0.0011899008435486756, "rouge2_recall": 0.12656441757990483, "rouge2_recall_stderr": 0.00313676095690092, "rougeL_fmeasure": 0.09186630288423041, "rougeL_fmeasure_stderr": 0.0019745485370318864, "rougeL_precision": 0.06178350535067078, "rougeL_precision_stderr": 0.0016362696987831417, "rougeL_recall": 0.2508928298018324, "rougeL_recall_stderr": 0.004573446137820672, "rougeLsum_fmeasure": 0.09105569938711074, "rougeLsum_fmeasure_stderr": 0.002004179378466674, "rougeLsum_precision": 0.06149602815890292, "rougeLsum_precision_stderr": 0.0016810240234989833, "rougeLsum_recall": 0.24666611039273245, "rougeLsum_recall_stderr": 0.004421778464668269}}, "1": {"PALM_prompt": {"bleu": 0.29130877143455586, "bleu_stderr": 0.0241959153769584, "rouge1_fmeasure": 0.09445919469734558, "rouge1_fmeasure_stderr": 0.001936296232548762, "rouge1_precision": 0.061208429338304574, "rouge1_precision_stderr": 0.0014402546745354069, "rouge1_recall": 0.28859748128262924, "rouge1_recall_stderr": 0.0049986158310419354, "rouge2_fmeasure": 0.04361066921501477, "rouge2_fmeasure_stderr": 0.001199759977794257, "rouge2_precision": 0.028263099280238797, "rouge2_precision_stderr": 0.0008784314903876122, "rouge2_recall": 0.13604193300293232, "rouge2_recall_stderr": 0.0032346419377748733, "rougeL_fmeasure": 0.09036600743610688, "rougeL_fmeasure_stderr": 0.0017702218609310719, "rougeL_precision": 0.05834596522178436, "rougeL_precision_stderr": 0.0012947018575173474, "rougeL_recall": 0.27784325004495686, "rougeL_recall_stderr": 0.0047400498630427775, "rougeLsum_fmeasure": 0.09022680216928644, "rougeLsum_fmeasure_stderr": 0.0018145691342534367, "rougeLsum_precision": 0.05847161583671669, "rougeLsum_precision_stderr": 0.0013503259579416152, "rougeLsum_recall": 0.27532137412331964, "rougeLsum_recall_stderr": 0.00463181262519618}}, "2": {"PALM_prompt": {"bleu": 0.26676093352000974, "bleu_stderr": 0.01745767099704443, "rouge1_fmeasure": 0.09217510177969336, "rouge1_fmeasure_stderr": 0.0018831049937955784, "rouge1_precision": 0.05941054322249251, "rouge1_precision_stderr": 0.0013961426249521755, "rouge1_recall": 0.2835970082022701, "rouge1_recall_stderr": 0.004788224891673027, "rouge2_fmeasure": 0.042424120797488585, "rouge2_fmeasure_stderr": 0.0011719244443792272, "rouge2_precision": 0.027316816074161285, "rouge2_precision_stderr": 0.0008486872680257794, "rouge2_recall": 0.1354275420032431, "rouge2_recall_stderr": 0.003183687122052663, "rougeL_fmeasure": 0.0876120055713282, "rougeL_fmeasure_stderr": 0.0017024934701633891, "rougeL_precision": 0.05626626520925391, "rougeL_precision_stderr": 0.001242253342941174, "rougeL_recall": 0.27095597028481616, "rougeL_recall_stderr": 0.004453368974145521, "rougeLsum_fmeasure": 0.08841779444595972, "rougeLsum_fmeasure_stderr": 0.0017834242097373026, "rougeLsum_precision": 0.05698004937770005, "rougeLsum_precision_stderr": 0.0013218209896847077, "rougeLsum_recall": 0.27207800066344545, "rougeLsum_recall_stderr": 0.004514833883372158}}, "3": {"PALM_prompt": {"bleu": 0.28612944934937573, "bleu_stderr": 0.030058507463981785, "rouge1_fmeasure": 0.09107726735341869, "rouge1_fmeasure_stderr": 0.0018176734285625148, "rouge1_precision": 0.05858674457588962, "rouge1_precision_stderr": 0.001331757166815278, "rouge1_recall": 0.27805772725490513, "rouge1_recall_stderr": 0.004621635420794643, "rouge2_fmeasure": 0.041642636541249896, "rouge2_fmeasure_stderr": 0.0011260991098264973, "rouge2_precision": 0.02669592353017647, "rouge2_precision_stderr": 0.0008008577341120715, "rouge2_recall": 0.13073693050689278, "rouge2_recall_stderr": 0.0030684516936530922, "rougeL_fmeasure": 0.08669913456777828, "rougeL_fmeasure_stderr": 0.0016464359763810076, "rougeL_precision": 0.0556444229114036, "rougeL_precision_stderr": 0.0011944680158197211, "rougeL_recall": 0.2654099060152725, "rougeL_recall_stderr": 0.004278095570672299, "rougeLsum_fmeasure": 0.08753387249247915, "rougeLsum_fmeasure_stderr": 0.0017181361371174542, "rougeLsum_precision": 0.05629907335785418, "rougeLsum_precision_stderr": 0.0012605456096568002, "rougeLsum_recall": 0.26728724980476815, "rougeLsum_recall_stderr": 0.004336618966824677}}, "4": {"PALM_prompt": {"bleu": 0.26705903217674426, "bleu_stderr": 0.02405825312974743, "rouge1_fmeasure": 0.09372278299065458, "rouge1_fmeasure_stderr": 0.0018460404864548486, "rouge1_precision": 0.06086053476185062, "rouge1_precision_stderr": 0.0014318320841098697, "rouge1_recall": 0.28542751510426556, "rouge1_recall_stderr": 0.004692910204968811, "rouge2_fmeasure": 0.04240340414142105, "rouge2_fmeasure_stderr": 0.0011357663748880447, "rouge2_precision": 0.027485650453758618, "rouge2_precision_stderr": 0.0008756611121258637, "rouge2_recall": 0.13419537296922995, "rouge2_recall_stderr": 0.003122488099367849, "rougeL_fmeasure": 0.08903710986289586, "rougeL_fmeasure_stderr": 0.0016750629187636654, "rougeL_precision": 0.057676657207041945, "rougeL_precision_stderr": 0.0012931118119002293, "rougeL_recall": 0.2722634215290626, "rougeL_recall_stderr": 0.00437854396102679, "rougeLsum_fmeasure": 0.09001878404549397, "rougeLsum_fmeasure_stderr": 0.0017466945253972782, "rougeLsum_precision": 0.058444776332263366, "rougeLsum_precision_stderr": 0.0013576642072023544, "rougeLsum_recall": 0.27420324540126445, "rougeLsum_recall_stderr": 0.0044098612417007715}}, "5": {"PALM_prompt": {"bleu": 0.32955887349530977, "bleu_stderr": 0.028674503871665574, "rouge1_fmeasure": 0.09759507774790617, "rouge1_fmeasure_stderr": 0.0019916427117499195, "rouge1_precision": 0.06531639829304567, "rouge1_precision_stderr": 0.0018118510618833096, "rouge1_recall": 0.2925041739048238, "rouge1_recall_stderr": 0.004813556224471976, "rouge2_fmeasure": 0.045510066340999515, "rouge2_fmeasure_stderr": 0.0012521676735104974, "rouge2_precision": 0.030671504391052696, "rouge2_precision_stderr": 0.0011385946802848516, "rouge2_recall": 0.14139940825924777, "rouge2_recall_stderr": 0.0032645688758915227, "rougeL_fmeasure": 0.09234102161235082, "rougeL_fmeasure_stderr": 0.001781843163970405, "rougeL_precision": 0.061620792950991723, "rougeL_precision_stderr": 0.0016331920676026356, "rougeL_recall": 0.2780347555856145, "rougeL_recall_stderr": 0.004448204648030782, "rougeLsum_fmeasure": 0.09340724567530662, "rougeLsum_fmeasure_stderr": 0.0018560344045322032, "rougeLsum_precision": 0.06243557617018187, "rougeLsum_precision_stderr": 0.0016873068321191943, "rougeLsum_recall": 0.2804738669361521, "rougeLsum_recall_stderr": 0.004518723891360148}}}, "GEM/wiki_lingua_en": {"0": {"tldr_en": {"bleu": 1.0659937881497992, "bleu_stderr": 0.05080391562111616, "rouge1_fmeasure": 0.14417929178250435, "rouge1_fmeasure_stderr": 0.001842074002833332, "rouge1_precision": 0.12608148311182882, "rouge1_precision_stderr": 0.001877790179736341, "rouge1_recall": 0.2049303749096855, "rouge1_recall_stderr": 0.0025880256283742705, "rouge2_fmeasure": 0.025455042866036937, "rouge2_fmeasure_stderr": 0.0007020843746137193, "rouge2_precision": 0.021892949427677305, "rouge2_precision_stderr": 0.0006292392217955498, "rouge2_recall": 0.037328795648641035, "rouge2_recall_stderr": 0.001109797618553952, "rougeL_fmeasure": 0.1172503924682141, "rougeL_fmeasure_stderr": 0.0013592087451697002, "rougeL_precision": 0.10138125064278208, "rougeL_precision_stderr": 0.0013701882816016511, "rougeL_recall": 0.17034733773499464, "rougeL_recall_stderr": 0.0020889089694998476, "rougeLsum_fmeasure": 0.13269967531471152, "rougeLsum_fmeasure_stderr": 0.0016802961690273025, "rougeLsum_precision": 0.11579468211486772, "rougeLsum_precision_stderr": 0.0017076861625592678, "rougeLsum_recall": 0.18934638686212538, "rougeLsum_recall_stderr": 0.002385952331190746}}, "1": {"tldr_en": {"bleu": 0.9957790449656027, "bleu_stderr": 0.02530207633623386, "rouge1_fmeasure": 0.146875208754622, "rouge1_fmeasure_stderr": 0.0016188663245510955, "rouge1_precision": 0.13086599178590658, "rouge1_precision_stderr": 0.001672077706598485, "rouge1_recall": 0.20720602482820027, "rouge1_recall_stderr": 0.0024533756253109274, "rouge2_fmeasure": 0.02035969927408505, "rouge2_fmeasure_stderr": 0.0006205740177121067, "rouge2_precision": 0.01792165073770399, "rouge2_precision_stderr": 0.0006428200847183632, "rouge2_recall": 0.03061200720033379, "rouge2_recall_stderr": 0.0011202861598998129, "rougeL_fmeasure": 0.10777539236344279, "rougeL_fmeasure_stderr": 0.0010799004074572, "rougeL_precision": 0.09572436493015317, "rougeL_precision_stderr": 0.0011588355930183143, "rougeL_recall": 0.1552970781871763, "rougeL_recall_stderr": 0.0018502784378049198, "rougeLsum_fmeasure": 0.13782959522383184, "rougeLsum_fmeasure_stderr": 0.0015003752806397137, "rougeLsum_precision": 0.12289889278902484, "rougeLsum_precision_stderr": 0.0015656539473863284, "rougeLsum_recall": 0.19479380254592224, "rougeLsum_recall_stderr": 0.00229544496822277}}, "2": {"tldr_en": {"bleu": 1.0261550894771894, "bleu_stderr": 0.03538435325757124, "rouge1_fmeasure": 0.14029812169934233, "rouge1_fmeasure_stderr": 0.0016746773304471378, "rouge1_precision": 0.1283280211841892, "rouge1_precision_stderr": 0.0018471880476730181, "rouge1_recall": 0.19588255520964618, "rouge1_recall_stderr": 0.0024808008313328897, "rouge2_fmeasure": 0.02068240392527567, "rouge2_fmeasure_stderr": 0.0006586430976373696, "rouge2_precision": 0.018646070313095908, "rouge2_precision_stderr": 0.0006575760758617186, "rouge2_recall": 0.030141933661931047, "rouge2_recall_stderr": 0.0010922246627428195, "rougeL_fmeasure": 0.10962969522414868, "rougeL_fmeasure_stderr": 0.0012104753879873735, "rougeL_precision": 0.09962500896923289, "rougeL_precision_stderr": 0.0013578472583766657, "rougeL_recall": 0.156123382155486, "rougeL_recall_stderr": 0.001955944209275228, "rougeLsum_fmeasure": 0.1309842621150673, "rougeLsum_fmeasure_stderr": 0.0015447855054700223, "rougeLsum_precision": 0.11997014021044887, "rougeLsum_precision_stderr": 0.0017299508610180456, "rougeLsum_recall": 0.18313685603265942, "rougeLsum_recall_stderr": 0.0023053627613329746}}, "3": {"tldr_en": {"bleu": 1.2439413140181725, "bleu_stderr": 0.08021377052333785, "rouge1_fmeasure": 0.12147663727978152, "rouge1_fmeasure_stderr": 0.0018368431033577758, "rouge1_precision": 0.1174937613247287, "rouge1_precision_stderr": 0.002215169887824098, "rouge1_recall": 0.16673838559605367, "rouge1_recall_stderr": 0.002632888074263044, "rouge2_fmeasure": 0.020106815196566395, "rouge2_fmeasure_stderr": 0.0006821011446245043, "rouge2_precision": 0.019684843075251483, "rouge2_precision_stderr": 0.0008735431083143301, "rouge2_recall": 0.028644261873361904, "rouge2_recall_stderr": 0.0010691604436933462, "rougeL_fmeasure": 0.09666295792962627, "rougeL_fmeasure_stderr": 0.0014039308878508574, "rougeL_precision": 0.09329668669102507, "rougeL_precision_stderr": 0.0017731591089089934, "rougeL_recall": 0.13511709856165047, "rougeL_recall_stderr": 0.0021379505707010355, "rougeLsum_fmeasure": 0.11304180944340365, "rougeLsum_fmeasure_stderr": 0.001697758818846021, "rougeLsum_precision": 0.10937771197761693, "rougeLsum_precision_stderr": 0.002069130170725252, "rougeLsum_recall": 0.15562267064023425, "rougeLsum_recall_stderr": 0.002460546591552193}}, "4": {"tldr_en": {"bleu": 0.2661907581558446, "bleu_stderr": 0.03202842695354204, "rouge1_fmeasure": 0.04098065480515757, "rouge1_fmeasure_stderr": 0.0014835204076038898, "rouge1_precision": 0.04156484323618865, "rouge1_precision_stderr": 0.0017463462545007212, "rouge1_recall": 0.05834009675533347, "rouge1_recall_stderr": 0.002218925403012533, "rouge2_fmeasure": 0.007236113220479305, "rouge2_fmeasure_stderr": 0.00045629416756844566, "rouge2_precision": 0.00736474207758501, "rouge2_precision_stderr": 0.0006168003872532538, "rouge2_recall": 0.010833168232581908, "rouge2_recall_stderr": 0.0007793950988884573, "rougeL_fmeasure": 0.033279909424772915, "rougeL_fmeasure_stderr": 0.0011802065589786679, "rougeL_precision": 0.034147189968935676, "rougeL_precision_stderr": 0.0014824990274979345, "rougeL_recall": 0.0480868680398638, "rougeL_recall_stderr": 0.0018328691284379799, "rougeLsum_fmeasure": 0.03810821262468117, "rougeLsum_fmeasure_stderr": 0.001380639279965432, "rougeLsum_precision": 0.03891404857872913, "rougeLsum_precision_stderr": 0.001653718902747482, "rougeLsum_recall": 0.054089485989470866, "rougeLsum_recall_stderr": 0.0020505430260237653}}, "5": {"tldr_en": {"bleu": 4.1838551572050286e-07, "bleu_stderr": 8.539089327004177e-07, "rouge1_fmeasure": 0.007348718460293708, "rouge1_fmeasure_stderr": 0.0007282980782455469, "rouge1_precision": 0.007397940774007664, "rouge1_precision_stderr": 0.0007992853925777316, "rouge1_recall": 0.010120982205394452, "rouge1_recall_stderr": 0.0010166711864938761, "rouge2_fmeasure": 0.001289849014264577, "rouge2_fmeasure_stderr": 0.00020396030875762924, "rouge2_precision": 0.00128680284734034, "rouge2_precision_stderr": 0.00022089770706856912, "rouge2_recall": 0.0018727716849492996, "rouge2_recall_stderr": 0.00035993430172752866, "rougeL_fmeasure": 0.0060987117278543486, "rougeL_fmeasure_stderr": 0.0005965491826266616, "rougeL_precision": 0.006134073610949074, "rougeL_precision_stderr": 0.000671334380732274, "rougeL_recall": 0.00856899865908427, "rougeL_recall_stderr": 0.0008562869548585046, "rougeLsum_fmeasure": 0.006898026781657591, "rougeLsum_fmeasure_stderr": 0.0006850161593107851, "rougeLsum_precision": 0.0069649726845215495, "rougeLsum_precision_stderr": 0.000761180679286655, "rougeLsum_recall": 0.009488299018839004, "rougeLsum_recall_stderr": 0.0009502861242391249}}}, "e2e_nlg_cleaned": {"0": {"generate_text_restaurant": {"bleu": 0.3337118408961487, "bleu_stderr": 0.05202728879470043, "rouge1_fmeasure": 0.0817116149040806, "rouge1_fmeasure_stderr": 0.00153613382148468, "rouge1_precision": 0.06853867384946524, "rouge1_precision_stderr": 0.0012188313815522228, "rouge1_recall": 0.1265014794105398, "rouge1_recall_stderr": 0.002527874019892803, "rouge2_fmeasure": 0.015367923182043539, "rouge2_fmeasure_stderr": 0.00045601328888639393, "rouge2_precision": 0.011247369789676413, "rouge2_precision_stderr": 0.00033868159563215023, "rouge2_recall": 0.025934933456238086, "rouge2_recall_stderr": 0.0007755385845050604, "rougeL_fmeasure": 0.08081655857040615, "rougeL_fmeasure_stderr": 0.0015247741746541353, "rougeL_precision": 0.06731071739815976, "rougeL_precision_stderr": 0.0011879498508703882, "rougeL_recall": 0.1256906248949207, "rougeL_recall_stderr": 0.0025226724722029028, "rougeLsum_fmeasure": 0.06255171257305257, "rougeLsum_fmeasure_stderr": 0.0011543628902571132, "rougeLsum_precision": 0.05420319385768004, "rougeLsum_precision_stderr": 0.0009667468979072273, "rougeLsum_recall": 0.09524325606060634, "rougeLsum_recall_stderr": 0.001900471716812503}}, "1": {"generate_text_restaurant": {"bleu": 8.580785815319153, "bleu_stderr": 0.12050962641485825, "rouge1_fmeasure": 0.38545334170928763, "rouge1_fmeasure_stderr": 0.002112571759353758, "rouge1_precision": 0.43028883128238876, "rouge1_precision_stderr": 0.0027777109499330867, "rouge1_recall": 0.3879649870783398, "rouge1_recall_stderr": 0.002764878871199931, "rouge2_fmeasure": 0.15599769101716504, "rouge2_fmeasure_stderr": 0.0016286782168266891, "rouge2_precision": 0.17625411435748609, "rouge2_precision_stderr": 0.002134450890393921, "rouge2_recall": 0.15821064491984946, "rouge2_recall_stderr": 0.0018746791495472526, "rougeL_fmeasure": 0.2737515668591034, "rougeL_fmeasure_stderr": 0.0016582477249727728, "rougeL_precision": 0.30818091438114104, "rougeL_precision_stderr": 0.0023544378053750622, "rougeL_recall": 0.27542498756422007, "rougeL_recall_stderr": 0.002136202440055506, "rougeLsum_fmeasure": 0.3182037827678809, "rougeLsum_fmeasure_stderr": 0.002003091072340103, "rougeLsum_precision": 0.35631742031115066, "rougeLsum_precision_stderr": 0.0026424737299773063, "rougeLsum_recall": 0.3201295543710165, "rougeLsum_recall_stderr": 0.002504264887357116}}, "2": {"generate_text_restaurant": {"bleu": 10.179948100725445, "bleu_stderr": 0.15921732874874983, "rouge1_fmeasure": 0.41061062362310147, "rouge1_fmeasure_stderr": 0.0021332461778007916, "rouge1_precision": 0.4701246692346356, "rouge1_precision_stderr": 0.002973249692601118, "rouge1_recall": 0.40133010839486466, "rouge1_recall_stderr": 0.002693499242079775, "rouge2_fmeasure": 0.18123862274918498, "rouge2_fmeasure_stderr": 0.0017413899623482422, "rouge2_precision": 0.21034647711506468, "rouge2_precision_stderr": 0.002262895180766459, "rouge2_recall": 0.17752340440327224, "rouge2_recall_stderr": 0.0019231348008012746, "rougeL_fmeasure": 0.2956205579339057, "rougeL_fmeasure_stderr": 0.00182234637449327, "rougeL_precision": 0.3398832332009184, "rougeL_precision_stderr": 0.002529788139802345, "rougeL_recall": 0.28899486933645147, "rougeL_recall_stderr": 0.002210430142434923, "rougeLsum_fmeasure": 0.33602585264600926, "rougeLsum_fmeasure_stderr": 0.0020441813309374983, "rougeLsum_precision": 0.3854788394930683, "rougeLsum_precision_stderr": 0.002765196187394664, "rougeLsum_recall": 0.3281945242802255, "rougeLsum_recall_stderr": 0.0024573282174066477}}, "3": {"generate_text_restaurant": {"bleu": 10.789731844190179, "bleu_stderr": 0.12017045252862463, "rouge1_fmeasure": 0.4160769563473889, "rouge1_fmeasure_stderr": 0.0021413655848613504, "rouge1_precision": 0.4829832996859665, "rouge1_precision_stderr": 0.0030484074957686427, "rouge1_recall": 0.4018418348119394, "rouge1_recall_stderr": 0.0026280744814466636, "rouge2_fmeasure": 0.18933351465139317, "rouge2_fmeasure_stderr": 0.0017958855840332813, "rouge2_precision": 0.22202793412057192, "rouge2_precision_stderr": 0.0023082250433674803, "rouge2_recall": 0.18324630983014636, "rouge2_recall_stderr": 0.00194920314491893, "rougeL_fmeasure": 0.30161525481953744, "rougeL_fmeasure_stderr": 0.0018405518808223482, "rougeL_precision": 0.35067227778777876, "rougeL_precision_stderr": 0.002546479986477843, "rougeL_recall": 0.2916905944938416, "rougeL_recall_stderr": 0.002177559260296222, "rougeLsum_fmeasure": 0.3437678815217894, "rougeLsum_fmeasure_stderr": 0.002090902101997608, "rougeLsum_precision": 0.399851493008314, "rougeLsum_precision_stderr": 0.0028835232605792824, "rougeLsum_recall": 0.331718034201341, "rougeLsum_recall_stderr": 0.002428440395770757}}, "4": {"generate_text_restaurant": {"bleu": 11.195536151651975, "bleu_stderr": 0.14805710268987346, "rouge1_fmeasure": 0.4214487218578573, "rouge1_fmeasure_stderr": 0.002125125404165521, "rouge1_precision": 0.4870389029505203, "rouge1_precision_stderr": 0.0030060268449702385, "rouge1_recall": 0.4067622516175034, "rouge1_recall_stderr": 0.002601979181985486, "rouge2_fmeasure": 0.19498627841697833, "rouge2_fmeasure_stderr": 0.0017901321774556802, "rouge2_precision": 0.22789485913434965, "rouge2_precision_stderr": 0.0022923772972932004, "rouge2_recall": 0.1884880182238668, "rouge2_recall_stderr": 0.0019387359058115024, "rougeL_fmeasure": 0.3049842055805278, "rougeL_fmeasure_stderr": 0.0018626579493065487, "rougeL_precision": 0.3527544385920249, "rougeL_precision_stderr": 0.0025426680941678494, "rougeL_recall": 0.2950536921267289, "rougeL_recall_stderr": 0.0022120407027767914, "rougeLsum_fmeasure": 0.34847134730498985, "rougeLsum_fmeasure_stderr": 0.002103441240841663, "rougeLsum_precision": 0.40283848434682135, "rougeLsum_precision_stderr": 0.0028488718557740248, "rougeLsum_recall": 0.3365908664066991, "rougeLsum_recall_stderr": 0.0024526373176337896}}, "5": {"generate_text_restaurant": {"bleu": 11.343200572605854, "bleu_stderr": 0.11916265276724415, "rouge1_fmeasure": 0.42600477331701564, "rouge1_fmeasure_stderr": 0.002088618061078895, "rouge1_precision": 0.4910389116851647, "rouge1_precision_stderr": 0.003058654910112927, "rouge1_recall": 0.41088969881696075, "rouge1_recall_stderr": 0.0025168338728330086, "rouge2_fmeasure": 0.1979775765065611, "rouge2_fmeasure_stderr": 0.0017651327620435838, "rouge2_precision": 0.23107977486062162, "rouge2_precision_stderr": 0.0023153275909847428, "rouge2_recall": 0.1911427232037733, "rouge2_recall_stderr": 0.001892959619131353, "rougeL_fmeasure": 0.3091536495871777, "rougeL_fmeasure_stderr": 0.0018398480793350466, "rougeL_precision": 0.35638430160338863, "rougeL_precision_stderr": 0.002569303538249397, "rougeL_recall": 0.29912750443223085, "rougeL_recall_stderr": 0.0021726065521660725, "rougeLsum_fmeasure": 0.3528767656515376, "rougeLsum_fmeasure_stderr": 0.0020652380895958976, "rougeLsum_precision": 0.4072928239657206, "rougeLsum_precision_stderr": 0.0028945881851336897, "rougeLsum_recall": 0.3403562650262804, "rougeLsum_recall_stderr": 0.0023737553325940625}}}, "gem_xsum": {"0": {"article_DOC_summary": {"bleu": 1.1792443767397391, "bleu_stderr": 0.0820008206453002, "rouge1_fmeasure": 0.18408359580247777, "rouge1_fmeasure_stderr": 0.0022265545774168786, "rouge1_precision": 0.13104806885016143, "rouge1_precision_stderr": 0.0016696061533377064, "rouge1_recall": 0.32207440240008434, "rouge1_recall_stderr": 0.003806583731509898, "rouge2_fmeasure": 0.033224548947791005, "rouge2_fmeasure_stderr": 0.0011698700374986966, "rouge2_precision": 0.023385645323327594, "rouge2_precision_stderr": 0.0008282879766144026, "rouge2_recall": 0.05994961316777745, "rouge2_recall_stderr": 0.0021516926902610404, "rougeL_fmeasure": 0.14066905424197854, "rougeL_fmeasure_stderr": 0.001589869491949137, "rougeL_precision": 0.0998987469408451, "rougeL_precision_stderr": 0.001181472858665789, "rougeL_recall": 0.2480871740164947, "rougeL_recall_stderr": 0.0028872095133784833, "rougeLsum_fmeasure": 0.14392005845390982, "rougeLsum_fmeasure_stderr": 0.001850757026152593, "rougeLsum_precision": 0.1022129966770997, "rougeLsum_precision_stderr": 0.0013625920887856532, "rougeLsum_recall": 0.2535042559665434, "rougeLsum_recall_stderr": 0.003307409271743902}}, "1": {"article_DOC_summary": {"bleu": 0.7333669023744487, "bleu_stderr": 0.13000332641058426, "rouge1_fmeasure": 0.14584396339276617, "rouge1_fmeasure_stderr": 0.002102637664735149, "rouge1_precision": 0.10336243780351441, "rouge1_precision_stderr": 0.0015685126380048779, "rouge1_recall": 0.2585747998651502, "rouge1_recall_stderr": 0.003590429754190019, "rouge2_fmeasure": 0.020919317079379954, "rouge2_fmeasure_stderr": 0.0009622499157490324, "rouge2_precision": 0.014720209928842373, "rouge2_precision_stderr": 0.000681021361327272, "rouge2_recall": 0.03777011619310744, "rouge2_recall_stderr": 0.0017739592882698512, "rougeL_fmeasure": 0.11612700909117789, "rougeL_fmeasure_stderr": 0.001545382459053668, "rougeL_precision": 0.0821069008768099, "rougeL_precision_stderr": 0.0011428484900625185, "rougeL_recall": 0.20740543017081056, "rougeL_recall_stderr": 0.0027830199070734097, "rougeLsum_fmeasure": 0.1161173925093962, "rougeLsum_fmeasure_stderr": 0.0016784793729300376, "rougeLsum_precision": 0.08210160844811192, "rougeLsum_precision_stderr": 0.0012398914712683133, "rougeLsum_recall": 0.20732540894587975, "rougeLsum_recall_stderr": 0.002979668668234706}}, "2": {"article_DOC_summary": {"bleu": 0.6768758898757194, "bleu_stderr": 0.053385621625246696, "rouge1_fmeasure": 0.1456451387246689, "rouge1_fmeasure_stderr": 0.002008514692784554, "rouge1_precision": 0.10294913437204971, "rouge1_precision_stderr": 0.001493869061603878, "rouge1_recall": 0.2593099601355609, "rouge1_recall_stderr": 0.003449879633572733, "rouge2_fmeasure": 0.019238022729627213, "rouge2_fmeasure_stderr": 0.0009148428605337303, "rouge2_precision": 0.013497535910280573, "rouge2_precision_stderr": 0.0006453882484652132, "rouge2_recall": 0.03494343970894335, "rouge2_recall_stderr": 0.0017020373666591004, "rougeL_fmeasure": 0.11593275835550473, "rougeL_fmeasure_stderr": 0.0015093999333849243, "rougeL_precision": 0.08180941551059284, "rougeL_precision_stderr": 0.0011179690230981208, "rougeL_recall": 0.20760572143774833, "rougeL_recall_stderr": 0.002691010842371602, "rougeLsum_fmeasure": 0.11626396730638126, "rougeLsum_fmeasure_stderr": 0.0016082221026397108, "rougeLsum_precision": 0.08204189444833064, "rougeLsum_precision_stderr": 0.0011860547696614754, "rougeLsum_recall": 0.20809616887189838, "rougeLsum_recall_stderr": 0.002858371023942416}}, "3": {"article_DOC_summary": {"bleu": 0.6820213665560544, "bleu_stderr": 0.05333356486252003, "rouge1_fmeasure": 0.13781067831884908, "rouge1_fmeasure_stderr": 0.0021174515290862607, "rouge1_precision": 0.09927106085703968, "rouge1_precision_stderr": 0.0016343150158814772, "rouge1_recall": 0.24287641032731458, "rouge1_recall_stderr": 0.0037267800062189636, "rouge2_fmeasure": 0.018121988245233207, "rouge2_fmeasure_stderr": 0.0009274218042565987, "rouge2_precision": 0.012812200577788478, "rouge2_precision_stderr": 0.0006581387038423858, "rouge2_recall": 0.03307527286460339, "rouge2_recall_stderr": 0.001751247026619044, "rougeL_fmeasure": 0.10987036544353425, "rougeL_fmeasure_stderr": 0.0016232442934585429, "rougeL_precision": 0.07901686664376605, "rougeL_precision_stderr": 0.001249903353276574, "rougeL_recall": 0.19452412633663918, "rougeL_recall_stderr": 0.002939914567539106, "rougeLsum_fmeasure": 0.11072488889531576, "rougeLsum_fmeasure_stderr": 0.0016889944652790365, "rougeLsum_precision": 0.07967747223853743, "rougeLsum_precision_stderr": 0.0013083430563035213, "rougeLsum_recall": 0.19619096905710873, "rougeLsum_recall_stderr": 0.003055668058044831}}, "4": {"article_DOC_summary": {"bleu": 0.30594617026264165, "bleu_stderr": 0.0676970728415138, "rouge1_fmeasure": 0.036965910785755694, "rouge1_fmeasure_stderr": 0.0020433261035716077, "rouge1_precision": 0.03251077542517423, "rouge1_precision_stderr": 0.00213991755859698, "rouge1_recall": 0.05897386947594356, "rouge1_recall_stderr": 0.0033433004286718858, "rouge2_fmeasure": 0.004171377402417491, "rouge2_fmeasure_stderr": 0.00048514268965227494, "rouge2_precision": 0.0032640836548389874, "rouge2_precision_stderr": 0.0004175254160021555, "rouge2_recall": 0.007119053240914698, "rouge2_recall_stderr": 0.0008708108424698816, "rougeL_fmeasure": 0.02943448947716526, "rougeL_fmeasure_stderr": 0.0016172466516820967, "rougeL_precision": 0.02634109151345856, "rougeL_precision_stderr": 0.0018307262109688949, "rougeL_recall": 0.04715698364480056, "rougeL_recall_stderr": 0.0026789790488207796, "rougeLsum_fmeasure": 0.03027547015903745, "rougeLsum_fmeasure_stderr": 0.0016766053945180429, "rougeLsum_precision": 0.027004915852553733, "rougeLsum_precision_stderr": 0.001864765148099242, "rougeLsum_recall": 0.04844961929509366, "rougeLsum_recall_stderr": 0.0027766936705316205}}, "5": {"article_DOC_summary": {"bleu": 8.851716353109152e-37, "bleu_stderr": 9.796533431961898e-31, "rouge1_fmeasure": 0.0023769191008089527, "rouge1_fmeasure_stderr": 0.000664374120380284, "rouge1_precision": 0.0026814100964168137, "rouge1_precision_stderr": 0.0007831104236380066, "rouge1_recall": 0.0022093761015055374, "rouge1_recall_stderr": 0.0006066899354878547, "rouge2_fmeasure": 0.0004044229735136974, "rouge2_fmeasure_stderr": 0.00020796051307837393, "rouge2_precision": 0.0005020375788966244, "rouge2_precision_stderr": 0.00027564515577918684, "rouge2_recall": 0.0003480638857997348, "rouge2_recall_stderr": 0.000171178861038322, "rougeL_fmeasure": 0.0018113358252006796, "rougeL_fmeasure_stderr": 0.0005084451143738701, "rougeL_precision": 0.002048403417496238, "rougeL_precision_stderr": 0.0006011286336239785, "rougeL_recall": 0.0016722359345367028, "rougeL_recall_stderr": 0.00045762893541433525, "rougeLsum_fmeasure": 0.0019434641482280037, "rougeLsum_fmeasure_stderr": 0.000531162691535884, "rougeLsum_precision": 0.002167331876280997, "rougeLsum_precision_stderr": 0.000612910003152478, "rougeLsum_recall": 0.0018286673479048533, "rougeLsum_recall_stderr": 0.0004992679324315411}}}} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a6cca8f5875273ac64423f66e56f953414f1a478 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.3099668612165794, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03007866592593964 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06460958235496345, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017809066736377858 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.25749913361086507, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004704092035936928 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09558920039471365, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.002149036988481146 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.031187491849803337, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0011899008435486756 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.12656441757990483, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.00313676095690092 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04582551071096432, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0013261745370059053 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.06178350535067078, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0016362696987831417 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2508928298018324, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004573446137820672 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09186630288423041, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0019745485370318864 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06149602815890292, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016810240234989833 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.24666611039273245, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004421778464668269 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09105569938711074, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002004179378466674 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.json new file mode 100644 index 0000000000000000000000000000000000000000..46ec5a0fc3ccd840f3bc67dd5653d1f4351fb9f1 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.29130877143455586, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.0241959153769584 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.061208429338304574, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014402546745354069 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.28859748128262924, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0049986158310419354 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09445919469734558, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001936296232548762 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.028263099280238797, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008784314903876122 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13604193300293232, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0032346419377748733 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04361066921501477, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.001199759977794257 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05834596522178436, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012947018575173474 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.27784325004495686, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0047400498630427775 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09036600743610688, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0017702218609310719 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05847161583671669, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013503259579416152 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.27532137412331964, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.00463181262519618 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09022680216928644, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018145691342534367 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.json new file mode 100644 index 0000000000000000000000000000000000000000..e9ce758252e6b495a39cf58d2af8ad62e91c0a6c --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.26676093352000974, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.01745767099704443 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.05941054322249251, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0013961426249521755 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2835970082022701, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004788224891673027 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09217510177969336, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018831049937955784 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.027316816074161285, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008486872680257794 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.1354275420032431, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003183687122052663 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.042424120797488585, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011719244443792272 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.05626626520925391, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.001242253342941174 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.27095597028481616, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004453368974145521 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.0876120055713282, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0017024934701633891 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05698004937770005, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013218209896847077 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.27207800066344545, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004514833883372158 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.08841779444595972, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017834242097373026 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.json new file mode 100644 index 0000000000000000000000000000000000000000..e35588de7685b78393531f3351bbd14e145542f5 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.28612944934937573, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.030058507463981785 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.05858674457588962, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001331757166815278 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.27805772725490513, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004621635420794643 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09107726735341869, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018176734285625148 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.02669592353017647, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008008577341120715 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13073693050689278, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0030684516936530922 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.041642636541249896, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011260991098264973 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.0556444229114036, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011944680158197211 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2654099060152725, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004278095570672299 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.08669913456777828, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016464359763810076 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.05629907335785418, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0012605456096568002 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.26728724980476815, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004336618966824677 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.08753387249247915, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017181361371174542 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.json new file mode 100644 index 0000000000000000000000000000000000000000..bf1db7fb3e820a9477955c2fbfddc577dda7b774 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.26705903217674426, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02405825312974743 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06086053476185062, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0014318320841098697 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.28542751510426556, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004692910204968811 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09372278299065458, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018460404864548486 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.027485650453758618, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008756611121258637 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.13419537296922995, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.003122488099367849 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.04240340414142105, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0011357663748880447 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.057676657207041945, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0012931118119002293 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2722634215290626, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.00437854396102679 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.08903710986289586, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0016750629187636654 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.058444776332263366, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0013576642072023544 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.27420324540126445, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0044098612417007715 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09001878404549397, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0017466945253972782 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.json new file mode 100644 index 0000000000000000000000000000000000000000..cf016d33a08cf85cd72659917e0fc3a06b3425f5 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-web_nlg_en_PALM_prompt_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "bleu": 0.32955887349530977, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.028674503871665574 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_precision": 0.06531639829304567, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0018118510618833096 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_recall": 0.2925041739048238, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.004813556224471976 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge1_fmeasure": 0.09759507774790617, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0019916427117499195 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_precision": 0.030671504391052696, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0011385946802848516 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_recall": 0.14139940825924777, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0032645688758915227 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rouge2_fmeasure": 0.045510066340999515, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0012521676735104974 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_precision": 0.061620792950991723, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0016331920676026356 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_recall": 0.2780347555856145, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.004448204648030782 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeL_fmeasure": 0.09234102161235082, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.001781843163970405 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_precision": 0.06243557617018187, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0016873068321191943 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_recall": 0.2804738669361521, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.004518723891360148 + }, + { + "task_name": "GEM/web_nlg_en", + "prompt_name": "PALM_prompt", + "rougeLsum_fmeasure": 0.09340724567530662, + "dataset_path": "GEM/web_nlg", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0018560344045322032 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8e3f15e6952c3d4b2232b0bee50de8108937dc1d --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.12608148311182882, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001877790179736341 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.2049303749096855, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0025880256283742705 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.14417929178250435, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.001842074002833332 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.021892949427677305, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006292392217955498 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.037328795648641035, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.001109797618553952 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.025455042866036937, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0007020843746137193 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.10138125064278208, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013701882816016511 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.17034733773499464, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0020889089694998476 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.1172503924682141, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0013592087451697002 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.11579468211486772, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0017076861625592678 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.18934638686212538, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002385952331190746 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.13269967531471152, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0016802961690273025 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.0659937881497992, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.05080391562111616 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f656921ca20aac1d8e43a94f50bdc13bdb0cfa31 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.13086599178590658, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.001672077706598485 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.20720602482820027, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0024533756253109274 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.146875208754622, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016188663245510955 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.01792165073770399, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006428200847183632 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.03061200720033379, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0011202861598998129 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.02035969927408505, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0006205740177121067 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.09572436493015317, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0011588355930183143 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.1552970781871763, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0018502784378049198 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.10777539236344279, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0010799004074572 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.12289889278902484, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0015656539473863284 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.19479380254592224, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.00229544496822277 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.13782959522383184, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015003752806397137 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.9957790449656027, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.02530207633623386 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.json new file mode 100644 index 0000000000000000000000000000000000000000..eaba21ec7c6dc360cc81afb177b8f83d450d901a --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.1283280211841892, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0018471880476730181 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.19588255520964618, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0024808008313328897 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.14029812169934233, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0016746773304471378 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.018646070313095908, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006575760758617186 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.030141933661931047, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0010922246627428195 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.02068240392527567, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0006586430976373696 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.09962500896923289, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0013578472583766657 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.156123382155486, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.001955944209275228 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.10962969522414868, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0012104753879873735 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.11997014021044887, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.0017299508610180456 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.18313685603265942, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0023053627613329746 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.1309842621150673, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0015447855054700223 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.0261550894771894, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03538435325757124 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.json new file mode 100644 index 0000000000000000000000000000000000000000..219500700e79920bb0355d223000b23509b9a6dc --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.1174937613247287, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.002215169887824098 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.16673838559605367, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002632888074263044 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.12147663727978152, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0018368431033577758 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.019684843075251483, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0008735431083143301 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.028644261873361904, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0010691604436933462 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.020106815196566395, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.0006821011446245043 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.09329668669102507, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0017731591089089934 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.13511709856165047, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0021379505707010355 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.09666295792962627, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0014039308878508574 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.10937771197761693, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.002069130170725252 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.15562267064023425, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.002460546591552193 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.11304180944340365, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001697758818846021 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 1.2439413140181725, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.08021377052333785 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ad8ed527e5f064457da1a0cf713a63025306a2e0 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.04156484323618865, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0017463462545007212 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.05834009675533347, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.002218925403012533 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.04098065480515757, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0014835204076038898 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.00736474207758501, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.0006168003872532538 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.010833168232581908, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.0007793950988884573 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.007236113220479305, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00045629416756844566 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.034147189968935676, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.0014824990274979345 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.0480868680398638, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0018328691284379799 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.033279909424772915, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0011802065589786679 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.03891404857872913, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.001653718902747482 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.054089485989470866, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0020505430260237653 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.03810821262468117, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.001380639279965432 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 0.2661907581558446, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 0.03202842695354204 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.json new file mode 100644 index 0000000000000000000000000000000000000000..98f08fc19d7b53ae86d5901e936f18dbd3e53467 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_GEM-wiki_lingua_en_tldr_en_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_precision": 0.007397940774007664, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_precision_stderr": 0.0007992853925777316 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_recall": 0.010120982205394452, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_recall_stderr": 0.0010166711864938761 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge1_fmeasure": 0.007348718460293708, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge1_fmeasure_stderr": 0.0007282980782455469 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_precision": 0.00128680284734034, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_precision_stderr": 0.00022089770706856912 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_recall": 0.0018727716849492996, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_recall_stderr": 0.00035993430172752866 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rouge2_fmeasure": 0.001289849014264577, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rouge2_fmeasure_stderr": 0.00020396030875762924 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_precision": 0.006134073610949074, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_precision_stderr": 0.000671334380732274 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_recall": 0.00856899865908427, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_recall_stderr": 0.0008562869548585046 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeL_fmeasure": 0.0060987117278543486, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeL_fmeasure_stderr": 0.0005965491826266616 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_precision": 0.0069649726845215495, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_precision_stderr": 0.000761180679286655 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_recall": 0.009488299018839004, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_recall_stderr": 0.0009502861242391249 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "rougeLsum_fmeasure": 0.006898026781657591, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0006850161593107851 + }, + { + "task_name": "GEM/wiki_lingua_en", + "prompt_name": "tldr_en", + "bleu": 4.1838551572050286e-07, + "dataset_path": "GEM/wiki_lingua", + "dataset_name": "en", + "subset": null, + "bleu_stderr": 8.539089327004177e-07 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0612e463de387eea972523ac745e2ef2fa4662ee --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 0.3337118408961487, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.05202728879470043 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.06853867384946524, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0012188313815522228 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.1265014794105398, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002527874019892803 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.0817116149040806, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.00153613382148468 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.011247369789676413, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.00033868159563215023 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.025934933456238086, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0007755385845050604 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.015367923182043539, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.00045601328888639393 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.06731071739815976, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0011879498508703882 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.1256906248949207, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0025226724722029028 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.08081655857040615, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0015247741746541353 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.05420319385768004, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0009667468979072273 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.09524325606060634, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.001900471716812503 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.06255171257305257, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0011543628902571132 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.json new file mode 100644 index 0000000000000000000000000000000000000000..661fdde566a30c5af81e971d2f155f562f5ae479 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 8.580785815319153, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.12050962641485825 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.43028883128238876, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0027777109499330867 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.3879649870783398, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002764878871199931 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.38545334170928763, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002112571759353758 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.17625411435748609, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002134450890393921 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.15821064491984946, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0018746791495472526 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.15599769101716504, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0016286782168266891 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.30818091438114104, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0023544378053750622 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.27542498756422007, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002136202440055506 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.2737515668591034, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0016582477249727728 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.35631742031115066, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0026424737299773063 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3201295543710165, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002504264887357116 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3182037827678809, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002003091072340103 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.json new file mode 100644 index 0000000000000000000000000000000000000000..98c4862779535dd9ee8f0f78fcc038c395ff1399 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 10.179948100725445, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.15921732874874983 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4701246692346356, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.002973249692601118 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.40133010839486466, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002693499242079775 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.41061062362310147, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0021332461778007916 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.21034647711506468, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.002262895180766459 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.17752340440327224, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0019231348008012746 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.18123862274918498, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017413899623482422 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3398832332009184, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002529788139802345 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.28899486933645147, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002210430142434923 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.2956205579339057, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.00182234637449327 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.3854788394930683, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.002765196187394664 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3281945242802255, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0024573282174066477 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.33602585264600926, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0020441813309374983 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.json new file mode 100644 index 0000000000000000000000000000000000000000..216468e4903232b3ce1774819ba981d11f98163e --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 10.789731844190179, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.12017045252862463 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4829832996859665, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0030484074957686427 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4018418348119394, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0026280744814466636 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4160769563473889, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.0021413655848613504 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.22202793412057192, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0023082250433674803 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.18324630983014636, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.00194920314491893 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.18933351465139317, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017958855840332813 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.35067227778777876, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002546479986477843 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.2916905944938416, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.002177559260296222 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.30161525481953744, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018405518808223482 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.399851493008314, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0028835232605792824 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.331718034201341, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.002428440395770757 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3437678815217894, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002090902101997608 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.json new file mode 100644 index 0000000000000000000000000000000000000000..149a27e4876a5b515e7e711185a35882c95a622f --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.195536151651975, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.14805710268987346 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4870389029505203, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.0030060268449702385 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.4067622516175034, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.002601979181985486 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.4214487218578573, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002125125404165521 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.22789485913434965, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0022923772972932004 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.1884880182238668, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.0019387359058115024 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.19498627841697833, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017901321774556802 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.3527544385920249, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.0025426680941678494 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.2950536921267289, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0022120407027767914 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3049842055805278, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018626579493065487 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.40283848434682135, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0028488718557740248 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3365908664066991, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0024526373176337896 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.34847134730498985, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.002103441240841663 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.json new file mode 100644 index 0000000000000000000000000000000000000000..e71aebdf739b119b433edde03b9e53f0bbdf1433 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_e2e_nlg_cleaned_generate_text_restaurant_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "bleu": 11.343200572605854, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "bleu_stderr": 0.11916265276724415 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_precision": 0.4910389116851647, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_precision_stderr": 0.003058654910112927 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_recall": 0.41088969881696075, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_recall_stderr": 0.0025168338728330086 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge1_fmeasure": 0.42600477331701564, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge1_fmeasure_stderr": 0.002088618061078895 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_precision": 0.23107977486062162, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_precision_stderr": 0.0023153275909847428 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_recall": 0.1911427232037733, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_recall_stderr": 0.001892959619131353 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rouge2_fmeasure": 0.1979775765065611, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rouge2_fmeasure_stderr": 0.0017651327620435838 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_precision": 0.35638430160338863, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_precision_stderr": 0.002569303538249397 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_recall": 0.29912750443223085, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_recall_stderr": 0.0021726065521660725 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeL_fmeasure": 0.3091536495871777, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeL_fmeasure_stderr": 0.0018398480793350466 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_precision": 0.4072928239657206, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_precision_stderr": 0.0028945881851336897 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_recall": 0.3403562650262804, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_recall_stderr": 0.0023737553325940625 + }, + { + "task_name": "e2e_nlg_cleaned", + "prompt_name": "generate_text_restaurant", + "rougeLsum_fmeasure": 0.3528767656515376, + "dataset_path": "e2e_nlg_cleaned", + "dataset_name": null, + "subset": null, + "rougeLsum_fmeasure_stderr": 0.0020652380895958976 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_0.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_0.json new file mode 100644 index 0000000000000000000000000000000000000000..74d2a095e8ff635dfd030bc56d839f47df7ab8c3 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_0.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.13104806885016143, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0016696061533377064 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.32207440240008434, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003806583731509898 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.18408359580247777, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0022265545774168786 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.023385645323327594, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0008282879766144026 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.05994961316777745, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0021516926902610404 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.033224548947791005, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0011698700374986966 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0998987469408451, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001181472858665789 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.2480871740164947, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0028872095133784833 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.14066905424197854, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.001589869491949137 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.1022129966770997, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0013625920887856532 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.2535042559665434, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003307409271743902 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.14392005845390982, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.001850757026152593 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 1.1792443767397391, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.0820008206453002 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_1.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9b860a652b1b5c9ca9b264aefa8bf288060d3bc9 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_1.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.10336243780351441, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0015685126380048779 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2585747998651502, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003590429754190019 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.14584396339276617, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002102637664735149 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.014720209928842373, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.000681021361327272 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03777011619310744, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0017739592882698512 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.020919317079379954, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009622499157490324 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.0821069008768099, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0011428484900625185 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.20740543017081056, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0027830199070734097 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11612700909117789, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.001545382459053668 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.08210160844811192, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0012398914712683133 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.20732540894587975, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.002979668668234706 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.1161173925093962, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016784793729300376 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.7333669023744487, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.13000332641058426 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 1, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_2.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_2.json new file mode 100644 index 0000000000000000000000000000000000000000..ba438fe49f21b4218fecb4ad7703af27143e7fa6 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_2.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.10294913437204971, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.001493869061603878 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.2593099601355609, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.003449879633572733 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.1456451387246689, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.002008514692784554 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.013497535910280573, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006453882484652132 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03494343970894335, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0017020373666591004 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.019238022729627213, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009148428605337303 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.08180941551059284, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0011179690230981208 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.20760572143774833, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.002691010842371602 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.11593275835550473, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0015093999333849243 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.08204189444833064, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0011860547696614754 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.20809616887189838, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.002858371023942416 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.11626396730638126, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016082221026397108 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.6768758898757194, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.053385621625246696 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 2, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_3.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_3.json new file mode 100644 index 0000000000000000000000000000000000000000..db9dddfb1ab8548bc9c768c7ec820033ba605930 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_3.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.09927106085703968, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0016343150158814772 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.24287641032731458, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0037267800062189636 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.13781067831884908, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0021174515290862607 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.012812200577788478, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0006581387038423858 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.03307527286460339, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.001751247026619044 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.018121988245233207, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.0009274218042565987 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.07901686664376605, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.001249903353276574 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.19452412633663918, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.002939914567539106 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.10987036544353425, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016232442934585429 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.07967747223853743, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.0013083430563035213 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.19619096905710873, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.003055668058044831 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.11072488889531576, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016889944652790365 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.6820213665560544, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.05333356486252003 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 3, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_4.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e1868d76ea061974c4dc6eb75c03492d34928224 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_4.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.03251077542517423, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.00213991755859698 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.05897386947594356, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0033433004286718858 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.036965910785755694, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.0020433261035716077 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0032640836548389874, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.0004175254160021555 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.007119053240914698, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.0008708108424698816 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.004171377402417491, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.00048514268965227494 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.02634109151345856, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0018307262109688949 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.04715698364480056, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.0026789790488207796 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.02943448947716526, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0016172466516820967 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.027004915852553733, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.001864765148099242 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.04844961929509366, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0027766936705316205 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.03027547015903745, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.0016766053945180429 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 0.30594617026264165, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 0.0676970728415138 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 4, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_5.json b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_5.json new file mode 100644 index 0000000000000000000000000000000000000000..ff923ae19825fa3ba5d146d4712cede39cdaa9a8 --- /dev/null +++ b/619m22b4b8/evaluation/generation/slim.619m22b4b8_gem_xsum_article_DOC_summary_5.json @@ -0,0 +1,133 @@ +{ + "results": [ + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_precision": 0.0026814100964168137, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_precision_stderr": 0.0007831104236380066 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_recall": 0.0022093761015055374, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_recall_stderr": 0.0006066899354878547 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge1_fmeasure": 0.0023769191008089527, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge1_fmeasure_stderr": 0.000664374120380284 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_precision": 0.0005020375788966244, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_precision_stderr": 0.00027564515577918684 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_recall": 0.0003480638857997348, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_recall_stderr": 0.000171178861038322 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rouge2_fmeasure": 0.0004044229735136974, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rouge2_fmeasure_stderr": 0.00020796051307837393 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_precision": 0.002048403417496238, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_precision_stderr": 0.0006011286336239785 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_recall": 0.0016722359345367028, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_recall_stderr": 0.00045762893541433525 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeL_fmeasure": 0.0018113358252006796, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeL_fmeasure_stderr": 0.0005084451143738701 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_precision": 0.002167331876280997, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_precision_stderr": 0.000612910003152478 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_recall": 0.0018286673479048533, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_recall_stderr": 0.0004992679324315411 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "rougeLsum_fmeasure": 0.0019434641482280037, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "rougeLsum_fmeasure_stderr": 0.000531162691535884 + }, + { + "task_name": "gem_xsum", + "prompt_name": "article_DOC_summary", + "bleu": 8.851716353109152e-37, + "dataset_path": "GEM/xsum", + "dataset_name": null, + "subset": "", + "bleu_stderr": 9.796533431961898e-31 + } + ], + "config": { + "model": "hf-causal", + "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-619m-22b/619m22b4b8/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", + "task_args": "", + "num_fewshot": 5, + "batch_size": 16, + "device": "cuda", + "use_cache": false, + "limit": 3000, + "bootstrap_iters": 10, + "seed": 1234 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_0.csv b/619m22b4b8/evaluation/rankeval/619m22b4b8_0.csv new file mode 100644 index 0000000000000000000000000000000000000000..0109a86283490262b8fa130df988d6fa4bc6682b --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_0.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.332,0.014899597242811485,0 +anli_r2,acc,0.334,0.014922019523732963,0 +anli_r3,acc,0.34,0.013680495725767785,0 +arc_challenge,acc,0.2030716723549488,0.011755899303705582,0 +arc_challenge,acc_norm,0.24744027303754265,0.012610352663292673,0 +arc_easy,acc,0.48569023569023567,0.010255580881603622,0 +arc_easy,acc_norm,0.41624579124579125,0.01011481940450088,0 +boolq,acc,0.5840978593272171,0.008620469604001021,1 +cb,acc,0.4107142857142857,0.0663363415035954,1 +cb,f1,0.2283333333333333,,1 +copa,acc,0.68,0.046882617226215034,0 +hellaswag,acc,0.3355905198167696,0.00471231451195096,0 +hellaswag,acc_norm,0.3990240987851026,0.004886969266944263,0 +piqa,acc,0.6833514689880305,0.010853160531978481,0 +piqa,acc_norm,0.6887921653971708,0.01080226387804584,0 +rte,acc,0.5342960288808665,0.030025579819366426,0 +sciq,acc,0.751,0.013681600278702315,0 +sciq,acc_norm,0.654,0.015050266127564448,0 +storycloze_2016,acc,0.6280064136825227,0.011177095517223677,0 +winogrande,acc,0.510655090765588,0.014049294536290393,0 diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_0.json b/619m22b4b8/evaluation/rankeval/619m22b4b8_0.json new file mode 100644 index 0000000000000000000000000000000000000000..147b36133973d03fc10e391165d28ef29c6e6842 --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_0.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.332, + "acc_stderr": 0.014899597242811485 + }, + "anli_r2": { + "acc": 0.334, + "acc_stderr": 0.014922019523732963 + }, + "anli_r3": { + "acc": 0.34, + "acc_stderr": 0.013680495725767785 + }, + "cb": { + "acc": 0.4107142857142857, + "acc_stderr": 0.0663363415035954, + "f1": 0.2283333333333333 + }, + "copa": { + "acc": 0.68, + "acc_stderr": 0.046882617226215034 + }, + "hellaswag": { + "acc": 0.3355905198167696, + "acc_stderr": 0.00471231451195096, + "acc_norm": 0.3990240987851026, + "acc_norm_stderr": 0.004886969266944263 + }, + "rte": { + "acc": 0.5342960288808665, + "acc_stderr": 0.030025579819366426 + }, + "winogrande": { + "acc": 0.510655090765588, + "acc_stderr": 0.014049294536290393 + }, + "storycloze_2016": { + "acc": 0.6280064136825227, + "acc_stderr": 0.011177095517223677 + }, + "boolq": { + "acc": 0.5840978593272171, + "acc_stderr": 0.008620469604001021 + }, + "arc_easy": { + "acc": 0.48569023569023567, + "acc_stderr": 0.010255580881603622, + "acc_norm": 0.41624579124579125, + "acc_norm_stderr": 0.01011481940450088 + }, + "arc_challenge": { + "acc": 0.2030716723549488, + "acc_stderr": 0.011755899303705582, + "acc_norm": 0.24744027303754265, + "acc_norm_stderr": 0.012610352663292673 + }, + "sciq": { + "acc": 0.751, + "acc_stderr": 0.013681600278702315, + "acc_norm": 0.654, + "acc_norm_stderr": 0.015050266127564448 + }, + "piqa": { + "acc": 0.6833514689880305, + "acc_stderr": 0.010853160531978481, + "acc_norm": 0.6887921653971708, + "acc_norm_stderr": 0.01080226387804584 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_1.csv b/619m22b4b8/evaluation/rankeval/619m22b4b8_1.csv new file mode 100644 index 0000000000000000000000000000000000000000..ae4fb6d4fc604779ddf2cba2310460384d3134a1 --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_1.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.334,0.014922019523732967,0 +anli_r2,acc,0.354,0.015129868238451772,0 +anli_r3,acc,0.3416666666666667,0.013696658778002505,0 +arc_challenge,acc,0.20648464163822525,0.011828865619002316,0 +arc_challenge,acc_norm,0.2440273037542662,0.012551447627856257,0 +arc_easy,acc,0.4797979797979798,0.010251405621305368,0 +arc_easy,acc_norm,0.42803030303030304,0.010152943316426268,0 +boolq,acc,0.5935779816513761,0.008590531708882184,1 +cb,acc,0.5,0.06741998624632421,1 +cb,f1,0.35057471264367807,,1 +copa,acc,0.68,0.04688261722621504,0 +hellaswag,acc,0.33479386576379205,0.0047095388649163305,0 +hellaswag,acc_norm,0.3974307906791476,0.004883663587184787,0 +piqa,acc,0.6784548422198041,0.010897500107575647,0 +piqa,acc_norm,0.6800870511425462,0.01088287358209206,0 +rte,acc,0.555956678700361,0.029907396333795997,0 +sciq,acc,0.778,0.013148721948877364,0 +sciq,acc_norm,0.737,0.013929286594259715,0 +storycloze_2016,acc,0.6194548369855692,0.011227604968407467,0 +winogrande,acc,0.5193370165745856,0.014041972733712965,0 diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_1.json b/619m22b4b8/evaluation/rankeval/619m22b4b8_1.json new file mode 100644 index 0000000000000000000000000000000000000000..0335e82b4058ac9081a4a30246a2aab859799d6e --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_1.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.334, + "acc_stderr": 0.014922019523732967 + }, + "anli_r2": { + "acc": 0.354, + "acc_stderr": 0.015129868238451772 + }, + "anli_r3": { + "acc": 0.3416666666666667, + "acc_stderr": 0.013696658778002505 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.35057471264367807 + }, + "copa": { + "acc": 0.68, + "acc_stderr": 0.04688261722621504 + }, + "hellaswag": { + "acc": 0.33479386576379205, + "acc_stderr": 0.0047095388649163305, + "acc_norm": 0.3974307906791476, + "acc_norm_stderr": 0.004883663587184787 + }, + "rte": { + "acc": 0.555956678700361, + "acc_stderr": 0.029907396333795997 + }, + "winogrande": { + "acc": 0.5193370165745856, + "acc_stderr": 0.014041972733712965 + }, + "storycloze_2016": { + "acc": 0.6194548369855692, + "acc_stderr": 0.011227604968407467 + }, + "boolq": { + "acc": 0.5935779816513761, + "acc_stderr": 0.008590531708882184 + }, + "arc_easy": { + "acc": 0.4797979797979798, + "acc_stderr": 0.010251405621305368, + "acc_norm": 0.42803030303030304, + "acc_norm_stderr": 0.010152943316426268 + }, + "arc_challenge": { + "acc": 0.20648464163822525, + "acc_stderr": 0.011828865619002316, + "acc_norm": 0.2440273037542662, + "acc_norm_stderr": 0.012551447627856257 + }, + "sciq": { + "acc": 0.778, + "acc_stderr": 0.013148721948877364, + "acc_norm": 0.737, + "acc_norm_stderr": 0.013929286594259715 + }, + "piqa": { + "acc": 0.6784548422198041, + "acc_stderr": 0.010897500107575647, + "acc_norm": 0.6800870511425462, + "acc_norm_stderr": 0.01088287358209206 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_2.csv b/619m22b4b8/evaluation/rankeval/619m22b4b8_2.csv new file mode 100644 index 0000000000000000000000000000000000000000..4292820c22ddffd5dbe9cc3045a0e5a199694e32 --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_2.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.339,0.014976758771620342,0 +anli_r2,acc,0.34,0.014987482264363937,0 +anli_r3,acc,0.325,0.013526454480351014,0 +arc_challenge,acc,0.20051194539249148,0.011700318050499378,0 +arc_challenge,acc_norm,0.2440273037542662,0.012551447627856255,0 +arc_easy,acc,0.4983164983164983,0.010259725364582781,0 +arc_easy,acc_norm,0.45075757575757575,0.010209906101011117,0 +boolq,acc,0.5941896024464832,0.008588486726385774,1 +cb,acc,0.4642857142857143,0.0672477765493766,1 +cb,f1,0.305982905982906,,1 +copa,acc,0.61,0.04902071300001975,0 +hellaswag,acc,0.33519219279028084,0.004710928569985747,0 +hellaswag,acc_norm,0.3972316271659032,0.00488324657949667,0 +piqa,acc,0.6713819368879217,0.010959127105167046,0 +piqa,acc_norm,0.6784548422198041,0.010897500107575656,0 +rte,acc,0.51985559566787,0.030072723167317184,0 +sciq,acc,0.804,0.01255952792670738,0 +sciq,acc_norm,0.757,0.013569640199177451,0 +storycloze_2016,acc,0.6231961517904864,0.011205964516279667,0 +winogrande,acc,0.5193370165745856,0.014041972733712969,0 diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_2.json b/619m22b4b8/evaluation/rankeval/619m22b4b8_2.json new file mode 100644 index 0000000000000000000000000000000000000000..8f5f2f5595d93a72f75e3cc402759ea69eeb02b9 --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_2.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.339, + "acc_stderr": 0.014976758771620342 + }, + "anli_r2": { + "acc": 0.34, + "acc_stderr": 0.014987482264363937 + }, + "anli_r3": { + "acc": 0.325, + "acc_stderr": 0.013526454480351014 + }, + "cb": { + "acc": 0.4642857142857143, + "acc_stderr": 0.0672477765493766, + "f1": 0.305982905982906 + }, + "copa": { + "acc": 0.61, + "acc_stderr": 0.04902071300001975 + }, + "hellaswag": { + "acc": 0.33519219279028084, + "acc_stderr": 0.004710928569985747, + "acc_norm": 0.3972316271659032, + "acc_norm_stderr": 0.00488324657949667 + }, + "rte": { + "acc": 0.51985559566787, + "acc_stderr": 0.030072723167317184 + }, + "winogrande": { + "acc": 0.5193370165745856, + "acc_stderr": 0.014041972733712969 + }, + "storycloze_2016": { + "acc": 0.6231961517904864, + "acc_stderr": 0.011205964516279667 + }, + "boolq": { + "acc": 0.5941896024464832, + "acc_stderr": 0.008588486726385774 + }, + "arc_easy": { + "acc": 0.4983164983164983, + "acc_stderr": 0.010259725364582781, + "acc_norm": 0.45075757575757575, + "acc_norm_stderr": 0.010209906101011117 + }, + "arc_challenge": { + "acc": 0.20051194539249148, + "acc_stderr": 0.011700318050499378, + "acc_norm": 0.2440273037542662, + "acc_norm_stderr": 0.012551447627856255 + }, + "sciq": { + "acc": 0.804, + "acc_stderr": 0.01255952792670738, + "acc_norm": 0.757, + "acc_norm_stderr": 0.013569640199177451 + }, + "piqa": { + "acc": 0.6713819368879217, + "acc_stderr": 0.010959127105167046, + "acc_norm": 0.6784548422198041, + "acc_norm_stderr": 0.010897500107575656 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_3.csv b/619m22b4b8/evaluation/rankeval/619m22b4b8_3.csv new file mode 100644 index 0000000000000000000000000000000000000000..13d31c11441d844421ad43e23c69d4d1e1cad57c --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_3.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.334,0.014922019523732956,0 +anli_r2,acc,0.341,0.014998131348402704,0 +anli_r3,acc,0.3458333333333333,0.013736245342311012,0 +arc_challenge,acc,0.2090443686006826,0.011882746987406446,0 +arc_challenge,acc_norm,0.2525597269624573,0.012696728980207706,0 +arc_easy,acc,0.49242424242424243,0.010258605792153323,0 +arc_easy,acc_norm,0.45286195286195285,0.010214087372211396,0 +boolq,acc,0.6067278287461774,0.00854350553741787,1 +cb,acc,0.44642857142857145,0.067031892279424,1 +cb,f1,0.3054373522458629,,1 +copa,acc,0.64,0.04824181513244218,0 +hellaswag,acc,0.3345947022505477,0.004708842600177428,0 +hellaswag,acc_norm,0.3979286994622585,0.004884702412456094,0 +piqa,acc,0.6860718171926007,0.010827928134189643,0 +piqa,acc_norm,0.6800870511425462,0.010882873582092058,0 +rte,acc,0.516245487364621,0.030080573208738064,0 +sciq,acc,0.795,0.012772554096113121,0 +sciq,acc_norm,0.754,0.013626065817750638,0 +storycloze_2016,acc,0.6269374665954035,0.011183612906093182,0 +winogrande,acc,0.516179952644041,0.014045126130978603,0 diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_3.json b/619m22b4b8/evaluation/rankeval/619m22b4b8_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9b819af4ba8462489969916aabc59b9fafcab09f --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_3.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.334, + "acc_stderr": 0.014922019523732956 + }, + "anli_r2": { + "acc": 0.341, + "acc_stderr": 0.014998131348402704 + }, + "anli_r3": { + "acc": 0.3458333333333333, + "acc_stderr": 0.013736245342311012 + }, + "cb": { + "acc": 0.44642857142857145, + "acc_stderr": 0.067031892279424, + "f1": 0.3054373522458629 + }, + "copa": { + "acc": 0.64, + "acc_stderr": 0.04824181513244218 + }, + "hellaswag": { + "acc": 0.3345947022505477, + "acc_stderr": 0.004708842600177428, + "acc_norm": 0.3979286994622585, + "acc_norm_stderr": 0.004884702412456094 + }, + "rte": { + "acc": 0.516245487364621, + "acc_stderr": 0.030080573208738064 + }, + "winogrande": { + "acc": 0.516179952644041, + "acc_stderr": 0.014045126130978603 + }, + "storycloze_2016": { + "acc": 0.6269374665954035, + "acc_stderr": 0.011183612906093182 + }, + "boolq": { + "acc": 0.6067278287461774, + "acc_stderr": 0.00854350553741787 + }, + "arc_easy": { + "acc": 0.49242424242424243, + "acc_stderr": 0.010258605792153323, + "acc_norm": 0.45286195286195285, + "acc_norm_stderr": 0.010214087372211396 + }, + "arc_challenge": { + "acc": 0.2090443686006826, + "acc_stderr": 0.011882746987406446, + "acc_norm": 0.2525597269624573, + "acc_norm_stderr": 0.012696728980207706 + }, + "sciq": { + "acc": 0.795, + "acc_stderr": 0.012772554096113121, + "acc_norm": 0.754, + "acc_norm_stderr": 0.013626065817750638 + }, + "piqa": { + "acc": 0.6860718171926007, + "acc_stderr": 0.010827928134189643, + "acc_norm": 0.6800870511425462, + "acc_norm_stderr": 0.010882873582092058 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_4.csv b/619m22b4b8/evaluation/rankeval/619m22b4b8_4.csv new file mode 100644 index 0000000000000000000000000000000000000000..c5479496c6c081c59fb54d31486a18b044e2c909 --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_4.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.336,0.014944140233795021,0 +anli_r2,acc,0.352,0.015110404505648666,0 +anli_r3,acc,0.3308333333333333,0.013588208070708999,0 +arc_challenge,acc,0.2030716723549488,0.011755899303705582,0 +arc_challenge,acc_norm,0.2525597269624573,0.012696728980207704,0 +arc_easy,acc,0.49074074074074076,0.010258024147860681,0 +arc_easy,acc_norm,0.44823232323232326,0.01020464512685693,0 +boolq,acc,0.6085626911314985,0.008536430524403957,1 +cb,acc,0.5,0.06741998624632421,1 +cb,f1,0.34575866188769416,,1 +copa,acc,0.67,0.04725815626252607,0 +hellaswag,acc,0.3344951204939255,0.004708494114573996,0 +hellaswag,acc_norm,0.3973312089225254,0.004883455188908969,0 +piqa,acc,0.6844396082698585,0.010843119201758943,0 +piqa,acc_norm,0.6784548422198041,0.010897500107575652,0 +rte,acc,0.5234657039711191,0.030063300411902652,0 +sciq,acc,0.797,0.012726073744598288,0 +sciq,acc_norm,0.761,0.013493000446937594,0 +storycloze_2016,acc,0.6221272047033671,0.01121221988713706,0 +winogrande,acc,0.5035516969218626,0.014052131146915848,0 diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_4.json b/619m22b4b8/evaluation/rankeval/619m22b4b8_4.json new file mode 100644 index 0000000000000000000000000000000000000000..4f5a1c9dead3670554a195d798c87c4a43cab75f --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_4.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.336, + "acc_stderr": 0.014944140233795021 + }, + "anli_r2": { + "acc": 0.352, + "acc_stderr": 0.015110404505648666 + }, + "anli_r3": { + "acc": 0.3308333333333333, + "acc_stderr": 0.013588208070708999 + }, + "cb": { + "acc": 0.5, + "acc_stderr": 0.06741998624632421, + "f1": 0.34575866188769416 + }, + "copa": { + "acc": 0.67, + "acc_stderr": 0.04725815626252607 + }, + "hellaswag": { + "acc": 0.3344951204939255, + "acc_stderr": 0.004708494114573996, + "acc_norm": 0.3973312089225254, + "acc_norm_stderr": 0.004883455188908969 + }, + "rte": { + "acc": 0.5234657039711191, + "acc_stderr": 0.030063300411902652 + }, + "winogrande": { + "acc": 0.5035516969218626, + "acc_stderr": 0.014052131146915848 + }, + "storycloze_2016": { + "acc": 0.6221272047033671, + "acc_stderr": 0.01121221988713706 + }, + "boolq": { + "acc": 0.6085626911314985, + "acc_stderr": 0.008536430524403957 + }, + "arc_easy": { + "acc": 0.49074074074074076, + "acc_stderr": 0.010258024147860681, + "acc_norm": 0.44823232323232326, + "acc_norm_stderr": 0.01020464512685693 + }, + "arc_challenge": { + "acc": 0.2030716723549488, + "acc_stderr": 0.011755899303705582, + "acc_norm": 0.2525597269624573, + "acc_norm_stderr": 0.012696728980207704 + }, + "sciq": { + "acc": 0.797, + "acc_stderr": 0.012726073744598288, + "acc_norm": 0.761, + "acc_norm_stderr": 0.013493000446937594 + }, + "piqa": { + "acc": 0.6844396082698585, + "acc_stderr": 0.010843119201758943, + "acc_norm": 0.6784548422198041, + "acc_norm_stderr": 0.010897500107575652 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_5.csv b/619m22b4b8/evaluation/rankeval/619m22b4b8_5.csv new file mode 100644 index 0000000000000000000000000000000000000000..911640c12d040d8a02cc2fbe1acb5d94a93b53bb --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_5.csv @@ -0,0 +1,21 @@ +task,metric,value,err,version +anli_r1,acc,0.339,0.014976758771620344,0 +anli_r2,acc,0.332,0.014899597242811476,0 +anli_r3,acc,0.335,0.013630871843821479,0 +arc_challenge,acc,0.19965870307167236,0.011681625756888693,0 +arc_challenge,acc_norm,0.24829351535836178,0.012624912868089758,0 +arc_easy,acc,0.4978956228956229,0.010259692651537035,0 +arc_easy,acc_norm,0.45707070707070707,0.010221897564256056,0 +boolq,acc,0.6119266055045871,0.008523130584760846,1 +cb,acc,0.5535714285714286,0.06703189227942394,1 +cb,f1,0.36856368563685643,,1 +copa,acc,0.68,0.04688261722621504,0 +hellaswag,acc,0.3339972117108146,0.00470674815212532,0 +hellaswag,acc_norm,0.39762995419239194,0.004884079750433892,0 +piqa,acc,0.6751904243743199,0.010926296238294034,0 +piqa,acc_norm,0.676278563656148,0.010916765010708762,0 +rte,acc,0.5703971119133574,0.02979666882912467,0 +sciq,acc,0.802,0.012607733934175315,0 +sciq,acc_norm,0.767,0.013374972519220062,0 +storycloze_2016,acc,0.6194548369855692,0.011227604968407471,0 +winogrande,acc,0.5240726124704025,0.01403618966539513,0 diff --git a/619m22b4b8/evaluation/rankeval/619m22b4b8_5.json b/619m22b4b8/evaluation/rankeval/619m22b4b8_5.json new file mode 100644 index 0000000000000000000000000000000000000000..b5b9972677203b5cebbbd55fede62ce759baae94 --- /dev/null +++ b/619m22b4b8/evaluation/rankeval/619m22b4b8_5.json @@ -0,0 +1,87 @@ +{ + "results": { + "anli_r1": { + "acc": 0.339, + "acc_stderr": 0.014976758771620344 + }, + "anli_r2": { + "acc": 0.332, + "acc_stderr": 0.014899597242811476 + }, + "anli_r3": { + "acc": 0.335, + "acc_stderr": 0.013630871843821479 + }, + "cb": { + "acc": 0.5535714285714286, + "acc_stderr": 0.06703189227942394, + "f1": 0.36856368563685643 + }, + "copa": { + "acc": 0.68, + "acc_stderr": 0.04688261722621504 + }, + "hellaswag": { + "acc": 0.3339972117108146, + "acc_stderr": 0.00470674815212532, + "acc_norm": 0.39762995419239194, + "acc_norm_stderr": 0.004884079750433892 + }, + "rte": { + "acc": 0.5703971119133574, + "acc_stderr": 0.02979666882912467 + }, + "winogrande": { + "acc": 0.5240726124704025, + "acc_stderr": 0.01403618966539513 + }, + "storycloze_2016": { + "acc": 0.6194548369855692, + "acc_stderr": 0.011227604968407471 + }, + "boolq": { + "acc": 0.6119266055045871, + "acc_stderr": 0.008523130584760846 + }, + "arc_easy": { + "acc": 0.4978956228956229, + "acc_stderr": 0.010259692651537035, + "acc_norm": 0.45707070707070707, + "acc_norm_stderr": 0.010221897564256056 + }, + "arc_challenge": { + "acc": 0.19965870307167236, + "acc_stderr": 0.011681625756888693, + "acc_norm": 0.24829351535836178, + "acc_norm_stderr": 0.012624912868089758 + }, + "sciq": { + "acc": 0.802, + "acc_stderr": 0.012607733934175315, + "acc_norm": 0.767, + "acc_norm_stderr": 0.013374972519220062 + }, + "piqa": { + "acc": 0.6751904243743199, + "acc_stderr": 0.010926296238294034, + "acc_norm": 0.676278563656148, + "acc_norm_stderr": 0.010916765010708762 + } + }, + "versions": { + "anli_r1": 0, + "anli_r2": 0, + "anli_r3": 0, + "cb": 1, + "copa": 0, + "hellaswag": 0, + "rte": 0, + "winogrande": 0, + "storycloze_2016": 0, + "boolq": 1, + "arc_easy": 0, + "arc_challenge": 0, + "sciq": 0, + "piqa": 0 + } +} \ No newline at end of file diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e57658a70ecb1bf4601cc24548c96b77d22d53ea --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54810803ee9293df952169a0dad3abd2f27440c0c6c7e1aa17d2344955be49e6 +size 116013719 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc9d440e42ed0c85f460b06ef46598c7c48cc2f4 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6457812c0af385fd54db7821043c1a8b30b19fcf34ce059877137d57d07a5e21 +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b309e0a8ad74c994284a2bfac884de699648b3a --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0de45ee0fc1e861377e0b037f61e9ecb48fe8e4e0ad05d9f69890957cf7cddff +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59b276860a3592a91a0bcc9a061a82c33be8bc08 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:015ed5fce7348ebcc85067dbbf445b0dc18aee3a0f49d96680b700f6669394ff +size 116013602 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5facfea0c11f03079276325cf2728c6665b272cf --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e855497c07cf9cfabb6e8108a28fb06ce73ab9f50e7f5c7f341832363209eb +size 116013858 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03957d4acb73ca68d84b01e5802f61b6c5026971 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eea156e116a884732d326e3553d6b87a42f98c3cf188d724fae8cc33f0897ef +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3de9f3244f985cc147e75b2229f3cf1c705fcfd --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13d70746220b68616a4b54ab59572ef0f3ce7d9b93067670674539b597f54d3d +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ccc1b513b2b71bab50e500e1278956134327a23b --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:317326a71b5884c643ae7f7a104d1bff1db845acf5c80defbaaeacf07fa49136 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..359e3e3b9704b1fae48fd6feb116d1b2081e0d06 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:369cb9fbc6b6432134fc7fb556731decab02787b9138a7036ed5ef6c69ca9868 +size 116013602 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1150f183a0eea1d3f69371c527167720ac49cae2 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bdcae25c80bc81e84787f679fb2204083b830cbe3fa72185d4b8d38b98227ec +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6d2ad7eb321e1127506128b63753b1f1226c97b --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8d86828fb9e67da032b76d458ec1b4fc53c58d9bfd758a4addc39d8ae0f2694 +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d91355e01c2628b7e47572699fdee5645314c5e6 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:051fdf51232e3df11b35fba46e5031a269d9f2f4ddb156c9cabfdf054d2636b0 +size 116013719 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9dd58236df5d49603e5824af9c9f25a5dc7fadb --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adfa8381db3c7778bd219f9ea58b29a7c78d80de355e1cb24a9295f882ab301f +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ceb1dfa50f88a8604dee66e469881c273b610af7 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf5f6a55a5e47af8450a4a2800325d2698272543839a5b5988d6b315768ea0a5 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0dc34fff5e1485428b850197275314b10b7f80d --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf2d74faf6d9a169179c9342bf91a259d18fa67f1d8a48e4ee4d68c2b115e16e +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29c6d07a230d03ac4a7c56bde703711734da14fa --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f60d862e982e6af821654728b238fdf25675888fd5269b968d44073476a21b98 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd6b0bc7c5881494865baf77f52e7830491e62b8 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f090a928a8f3fa3b9fc568640971904e0f831183578948a05665a19c574adfc +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91d78bc69a114508a13d123bbbcf304f196ec065 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ea68ff923ffc4c597e52d071872627942eaa2b1bd58170786ce7191c3c75279 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..915694b533f7a9145ba8b35c1e5bd16ef26dfee0 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb04c3be840f291c77b7a4e0c64c924f56efad5a495a13f71ed6a70884c17fb +size 116013858 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ccd0ad300832e03fe91e34fd26a546442182c713 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:603fee7d8bc6d0ffdfd41aabd63983655e4f04293579c2e681853f91f1110e81 +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62a540205220dbe62c78814c6630e5c05e09bf45 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f0452adeb763ff0e87e8806931eceac94233165e916712db43b115a83ec08c1 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcd29275db90112393f2817175449e10ab0a04a1 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e6631b1ea0e2691ed99c835d3cc1a7d6c0b7049f2530b76271e2a0a32d8b1e9 +size 116013602 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3afa9a2ab15a4f7aaeee1b1cd55d73c2766d990b --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:869cac4e6bdd1234f4f624f4fb5727c66fe9f3bb3449c6f8c6986966878970b0 +size 116013655 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3998597e43ab0a5dec6cbf95cf02208dbb3f726 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49601849ae0ef2611e5f2fd5172c7ebf5d7d53f979fd3ed4274a2f28545a4cb0 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67c7961cfba06cf9717141d20ada35e737ee1f61 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:315b2dc768c9ecf8839da4ca4de9508f062fe0ee93b88ccf61ae4cc8e9742b28 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b48132497a4741df8fefeff8ccde6d41878643c5 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:173a30012caacb4b3af3570fb64904a01e2acf36af1866e4e482c4cbc363d693 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfd1a630dcd5aa8fc11668fa1e93c4efe52395cc --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b5e1672470559f2f6d8d1c5d30ff158490b56c4b6b34a7606de55aeaa5db658 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4d1de3b6544d8d5ae211d845c64c3c6ce73bcaf --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37d937f5f2eb666fd731d33d86a5649a571b4b65e8845b273f98c0cca8e0ee51 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..215cbc586f8a237949fd0484b0d4220021435d52 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fac82520c8cc341a4905dd2d1352fc941ce63f9a68f13aa6580ca3ce4757be6 +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a07b217d53fa4e80d06794b54f561419d970c2eb --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54a04821554879cf2b4f86833a405c0bdaa44589ff25bc216dd4e077c374fbc7 +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb0ccd33266087055d93ee5da69004b45036b7e7 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f72412ccb446c070914fc749f005355f4d193a4cc44d871e1b62def794e9cd4 +size 116013858 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14ec29619a6310d90b907412da48de563f3200a7 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09fc2f7729cbcaba45850c37a6b1ed3d1cb8a6f4f7903699b5f0b5527c7b7e4a +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3497269f560f6bb511e8a4a06eb8bf2f23179cea --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b30ce69161acd294bb19fe7d87207f6c7fa9ba515de79e2deb11e16b6ef20c4 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77744880d24d60e462280586a2701529f63b39ac --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b109250631351f0e6e9f8b7cb5db9fa9e00a11f3513a1f48885491080f6cb41 +size 116013719 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1654abdd57c7c493585d243792f5ee4c2313f2a9 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5faf6b6cf0fc03040f65561a6fc27bb4a09824fa65a58ae75a324568dec0892 +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6d172f684c7c12482ac255c8c70532a270b5db1 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:271c8776b954ebf06da0d7ec0157f5b926c989b6c37bb0fa35ec6c8ad25df6bd +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2dc2b5377c2e8a87e46524c90d3eb3332e98355 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:422a242c2dabe2ea67423285d9162c53832e94a7cc95bb954944deed9e989ec4 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f40934c93fecc900555881f17d3873fc9d3ca3f --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a138973cf542e12d19828089889a880ade8f90fa5a7fe661ac34ab266bd467b6 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38163c5da9b08ca43b7d2b082d664a03c7e6ba11 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d31752a8104c01aebb0e59dfac4386440267f10515a407704d1538c07dbadb +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a5b28d9608e4239dea24d6d9b6d5e4a7638ffe9 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:060da109908f3e16983d26a7a548d13d6a4277e65964510a859e7fbf5199906b +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..767da9ba323b44492080f454ae78a5c0c54ceeb5 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d908c656357a4be31e02a5e8620b91183ef58435c0ba4dd1614757c6eac374 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb41b565eceb0dc62d1066b8792f0a50cf38339a --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad718b7bf10e7ab656aa8c12b7c88ec1392ed508b311975cf6087d89f96af26b +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce738d1c0edd31cd11fe1e34f91ba2b975be3338 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4f56f9cf12f3dfa371a621be2522d2798802bae367660acd21e3634be53a2d +size 116013922 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff05dfa748561e5102e83f488b59bad2b0309df7 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18b86a6f95be6fda9791c7e7cb37f21913312dffbaa6f96965f82dff370a4e3f +size 116013538 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cea10c5d2749a68cdd2c6678403ec25242a82b3 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77d609a81f488a6f5214a92362d6cddd79b95f50582f2f74b9fa5abb9a320c2a +size 116013719 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f72c0d6f5a7f4b704e9a9a721c375b28b62c7e78 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6861fca8df6c7300491ad5247f3c697c13b0c54a78d6d9d5f1ef36b98a6ff28 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ca1bbcf2e91ece3c0dec16bea3571f59a959a15 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe92c059f591d5b8b65c6dcb4b8be5931ac1c23b1b480363b526ab71befaf8c +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1dadf9b6edb1fdd60eb688752f9c49c9c037edc8 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1767a8db57e19c9941c73f785c4ebe627f93021ddf8d351e0190d33c40296b09 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44f4dec4d590d66e99caa766d4d41f14ab114e98 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81da33fbafb3e343cb8cf3b8285300f93b0e84d434d7842f495a96188c9399dd +size 116013922 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c63b358a0a870942b3377e36d3aa0b19ebcafde --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff2c77011a2110b9f795329bee8c4859ed3a628c6282f4f8ae8715cd744d5e18 +size 116013602 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0864f788657239cedfd610d8006d9cee22d711d6 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f86fccaca243c4d592ea80a67192ab002e2618d27fe81fa9c4042c8862a90cd +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efed9d92138f7b3132367740a60bf9990d0931d0 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9918b5fad4d52402000a7e5960c587021318669593bc28d24427f4f88c49b2a +size 116013602 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06e9f42a8ef3aad225744aa83358b4923d2eb4aa --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8d2c0f3f2af4ddb5532fd24aa95198b451e7b8a8679af038d05aa8148788482 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1255e130cf033794f6414ca114bfb2a9c7a34243 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34308266fcbdb5d5e255dbe91160a6f466a120dac0faf4256a70a370e8247346 +size 116013858 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5cca11ca00ed3938ae5d3f0702583ed79d1ca0a --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:348c02b36fe613938ce6f269d40517008ef8857bc10df4db8a7ae73a13cfdb09 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f70b388b7006481a1a6ffb25a6c0848ec3a65ea --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a337b15a00a908613b96574e3c15d658cfe7ff241c1bc75d287de999bbcd1790 +size 116013655 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70e5a430c0de233e1ac222aadbe1b719d4744b43 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4531d5304498caa824d6f6abfb0ad3e9c626d0e93ff7602b4fa9d37017558b90 +size 116013730 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fee011ec728ddf24f6e93e0a6dc3e09ff5278fb1 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d130d3f481a14f89534f5ef5a4d417eb20bb0153bc51d3df0b440c763745b5d7 +size 116013794 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d77f194461d2b1dcb30870826a7603649ba1924 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42a8c17f849ca0553365ddd8d0220ae23b893082e1ed09ba58ed5e28f88bd2ac +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..283197dd10facf37ba083abfc6455525b873c836 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88e8ac7bb61183c3ad0db922c78711fad6d9ef397c38f36744dc156009e4010e +size 116013666 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6d4fc5f646bd4b2c187e51dcc5db4d8c815d21f --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d642a8921b4a2de2b7e08faee28f4c96b3e44456c7a722c903cd4e68bd20969a +size 116013719 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..930a27ad3cc2cf701e690a24248b21e4ecdf5c54 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e563baec2088f2b11c5c74b2bffe0fa2de77d133cc61bdc4a76487edca2caa44 +size 116013719 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..053497446e9ba58a43d20e2a55ae50d9976019fe --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:670c4a4d0b9137f9c509d62d99df8168965e531e64ff66449d90ae707c945167 +size 116013655 diff --git a/619m22b4b8/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/619m22b4b8/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bb4cd48b38f7d71764ee7d84769ca6d9b9f9ff8 --- /dev/null +++ b/619m22b4b8/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e605c9eb318ae36e160730941bfb7f295c93c68b4a5600eace3b627d8eb114b5 +size 116013591 diff --git a/619m22b4b8/global_step41007/layer_01-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..386cf7dfec1df3a93c149b4c40f008af55c99138 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e606a5f3ce43782e47a2ad30e843ebc7ecd3885427291830cf69d0623e2761 +size 160826627 diff --git a/619m22b4b8/global_step41007/layer_03-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5ae9a5d64ba02ad642e85d43eb1dab8ebd033c8 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed26d7a8013683be04442b89cc2ba83a53df7cd91d4fa80511a0444db42d097d +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_04-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f13149af4b4cedd08e2fe0122c0c9bbde6d6989e --- /dev/null +++ b/619m22b4b8/global_step41007/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3969b368132abb8120cd6836cd758cdf7d1b0cf947face271b70e5c4cad8cd7 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_05-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a695ead88ca4ea80da65d1547dde3eb265c6b82e --- /dev/null +++ b/619m22b4b8/global_step41007/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00d40e90a96b01430438dc0c03e6a65ab1104b9418fddbea4ca98530f6cc610b +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_06-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3968a63e2d6a6436d66607af2052f6478a7a28a --- /dev/null +++ b/619m22b4b8/global_step41007/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9af9cb1aa442d71b04751fc855a831a45dacb6b7669cb3f32e74aeec27256d4d +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_07-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f15e3b256f1c7b180803f3acbce5fa3fc443217f --- /dev/null +++ b/619m22b4b8/global_step41007/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aa4f04c7088f056701ecbab66e7a00ad35183c330af9cfe9e23cfa2dc348b9a +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_08-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19ae3c248a44c516c15c9d0e12ea6cf560930ee6 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf27370492b1d3ecb98f534afc5174514322dc786dea46278fabaec86c76353e +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_09-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6598be865f643540c4860935b42247caae6d73c --- /dev/null +++ b/619m22b4b8/global_step41007/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad51f7d6b9d4a4d04dd6745f6be35b0199ab23abe735542d43b2c80cc4cc8f19 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_10-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac4ca285b0c9f20251791cf4a0dc26cc57af9c48 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aaa61e07655ff15d3704950b45d44b38e5cc72dc40983f7364f63c576bc0c93 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_11-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d19c9d30141d42b55f14c1ef430806c030c48c2 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e316d7016302009866b680dcf50dc04267f01a202212a868d1d44dda7f764084 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_12-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8dbd3a973b2e3f5f603f50dd3e8b22e50f4dcfb0 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0db9db1b614bc6042a196910f58f4262bcd894444b0653cfebc5bfd1a407909d +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_13-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58603f42a05a2008c7074c11c7690ca5b8b1d3ee --- /dev/null +++ b/619m22b4b8/global_step41007/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95f7d98537d991a3630fa2a453c1bdd8207d000e3e514d219177e78108173aaf +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_14-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aefe39090a1659aae94d4c040243d91adf1705b5 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f06d8b75ddfd9951e4284a5eaffa4aa1b9db3724b39706464da08c377137d86 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_15-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..088ab678ee426da8882cef3fb361b558b4098c3e --- /dev/null +++ b/619m22b4b8/global_step41007/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bad392b896ce15c549dbffa9e5205b9db9dbcd265aab1d7b61517a914393fc4 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_16-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e6923654490be1058000ad02827e6a599ff25c4 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:588be78e7ac07df93241a8bbc2c47cd3cd440e08d5e65f8af54c14333ef51c31 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_17-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c003426e04af0494c7928eb0f62565db98cc940f --- /dev/null +++ b/619m22b4b8/global_step41007/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b58a3378c52f8d04876bdb0c907f5ebab75387e62fba2bc9aafe0253c23e27ec +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_18-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0a79d7b4761b086107782a4724a235ca4fa284d --- /dev/null +++ b/619m22b4b8/global_step41007/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3f9130b0fb066a618264152b89630ddccfb67d94058e50e8e5ebf395b6fb6c6 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_19-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffb285f8f793f752fab233043d230d0d5f24e3fb --- /dev/null +++ b/619m22b4b8/global_step41007/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c3f802c231a8d9e6a6b642b6eaa886253de566c7f709f1b4c8dc37f4003421 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_20-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44733b95c16cc116cf7a38b9458b9fb8705b51fa --- /dev/null +++ b/619m22b4b8/global_step41007/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437c79dffa409c42d8085dbd2138685b27f8852dd3cedde714e37810e9dae4b9 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_21-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12d37adea45bcb30e5f61839b882d12ce49f8540 --- /dev/null +++ b/619m22b4b8/global_step41007/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:793a63109afd557dbe9a023690816091e9e951095d1cdc48ae8d7246843dd683 +size 56667395 diff --git a/619m22b4b8/global_step41007/layer_23-model_00-model_states.pt b/619m22b4b8/global_step41007/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe6514d2186dd99b4aa0ddfec4dd55d8f7866d1f --- /dev/null +++ b/619m22b4b8/global_step41007/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6383d6e5a6b5d6ef6aba4cc2cefa77909bef9780e69ad2ab23ac48e5d777d4b9 +size 7363 diff --git a/619m22b4b8/global_step41007/mp_rank_00_model_states.pt b/619m22b4b8/global_step41007/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7297a8d1eca37a80d45bb1a37c1ec8b38e40043 --- /dev/null +++ b/619m22b4b8/global_step41007/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75a85c363d699f3461b780f297fc1ff7e0dda29f6365bf471f5928ce46ef8b65 +size 38515 diff --git a/619m22b4b8/logs/2817194.err b/619m22b4b8/logs/2817194.err new file mode 100644 index 0000000000000000000000000000000000000000..7a3a8d5bb3aa7e32a5720163c1284e00850d0b66 --- /dev/null +++ b/619m22b4b8/logs/2817194.err @@ -0,0 +1,1118 @@ +0: rm: cannot remove '/dev/shm/hsakmt_shared_mem': Operation not permitted +0: rm: cannot remove '/dev/shm/sem.hsakmt_semaphore': Operation not permitted +4: rm: cannot remove '/dev/shm/hsakmt_shared_mem': Operation not permitted +1: rm: cannot remove '/dev/shm/hsakmt_shared_mem': Operation not permitted +3: rm: cannot remove '/dev/shm/hsakmt_shared_mem': Operation not permitted +3: rm: cannot remove '/dev/shm/sem.hsakmt_semaphore': Operation not permitted +4: rm: cannot remove '/dev/shm/sem.hsakmt_semaphore': Operation not permitted +1: rm: cannot remove '/dev/shm/sem.hsakmt_semaphore': Operation not permitted +2: rm: cannot remove '/dev/shm/hsakmt_shared_mem': Operation not permitted +2: rm: cannot remove '/dev/shm/sem.hsakmt_semaphore': Operation not permitted +6: rm: cannot remove '/dev/shm/hsakmt_shared_mem': Operation not permitted +6: rm: cannot remove '/dev/shm/sem.hsakmt_semaphore': Operation not permitted +0: 2023-02-09 15:51:29.633452: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:29.633200: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:29.633229: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633565: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633584: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633590: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: 2023-02-09 15:51:29.633487: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:51:29.633495: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633576: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633597: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633611: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: 2023-02-09 15:51:29.633523: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:51:29.633532: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:51:29.633530: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: 2023-02-09 15:51:29.633209: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:29.633233: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:29.633252: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: 2023-02-09 15:51:29.633643: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:51:29.633654: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:51:29.633662: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633623: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:51:29.633669: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:51:29.633679: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633644: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633609: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633659: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: 2023-02-09 15:51:29.633549: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:29.633268: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:29.633274: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:51:29.633682: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +0: 2023-02-09 15:51:29.633529: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:29.633366: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:51:29.633692: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +3: 2023-02-09 15:51:29.633667: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +6: 2023-02-09 15:51:29.633735: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633622: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633649: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633721: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633724: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +5: 2023-02-09 15:51:29.633749: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634263: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634278: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634290: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634284: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634284: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634277: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634266: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +7: 2023-02-09 15:51:29.634319: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889412: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889414: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889436: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889422: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889423: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889423: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889430: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2: 2023-02-09 15:51:30.889429: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.889992: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.889999: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.889999: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.890006: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.890002: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.889990: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.890012: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +1: 2023-02-09 15:51:30.890008: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +4: 2023-02-09 15:51:41.515164: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515832: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:51:41.515205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515844: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:51:41.515210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515862: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:51:41.515223: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515869: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:51:41.515876: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:51:41.515248: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 15:51:41.515886: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:51:41.515893: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 15:51:41.515901: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:51:41.516020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:51:41.516045: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 2023-02-09 15:51:41.516059: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:51:41.516041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:51:41.515934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 2023-02-09 15:51:41.516085: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 2023-02-09 15:51:41.516136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:51:41.516117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:51:41.516153: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 2023-02-09 15:51:41.516133: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:51:41.516165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-09 15:51:41.515960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 2023-02-09 15:51:41.516138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516087: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:51:41.516265: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:51:41.516180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-09 15:51:41.515979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 2023-02-09 15:51:41.516023: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 2023-02-09 15:51:41.516101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516073: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:51:41.516176: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:51:41.515989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:51:41.516154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516721: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:51:41.516296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 15:51:41.516290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 2023-02-09 15:51:41.516049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:51:41.516742: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:51:41.516748: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:51:41.516189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:51:41.516009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 15:51:41.516147: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516759: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:51:41.516616: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:51:41.516766: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:51:41.516194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 2023-02-09 15:51:41.516306: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-09 15:51:41.516006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 2023-02-09 15:51:41.516327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 15:51:41.516770: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:51:41.516630: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +5: 2023-02-09 15:51:41.516779: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +5: 2023-02-09 15:51:41.516783: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:51:41.516182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 2023-02-09 15:51:41.516344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-09 15:51:41.516025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 2023-02-09 15:51:41.516344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 2023-02-09 15:51:41.516830: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:51:41.516326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-09 15:51:41.516033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 2023-02-09 15:51:41.516391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:51:41.516847: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +0: 2023-02-09 15:51:41.516859: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:51:41.516877: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:51:41.516883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:51:41.516887: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +0: 2023-02-09 15:51:41.516889: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:51:41.516344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-09 15:51:41.516807: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.516385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 15:51:41.516894: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:51:41.516644: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:51:41.516826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +1: 2023-02-09 15:51:41.516362: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +3: 2023-02-09 15:51:41.516836: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.516405: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 15:51:41.516857: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:51:41.516859: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516928: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:51:41.516369: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +2: 2023-02-09 15:51:41.516390: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 2023-02-09 15:51:41.516945: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 15:51:41.516657: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:51:41.516879: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:51:41.516879: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +3: 2023-02-09 15:51:41.516883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516958: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:51:41.516660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:51:41.516670: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:51:41.516669: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +1: 2023-02-09 15:51:41.516674: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.516442: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200 +6: 2023-02-09 15:51:41.516981: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 15:51:41.516990: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:51:41.516993: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.516970: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.516995: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.517005: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.517001: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.517004: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.517011: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.517019: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:51:41.517026: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +2: 2023-02-09 15:51:41.517020: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +6: 2023-02-09 15:51:41.517041: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517342: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517360: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517370: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517382: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517388: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517391: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517398: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +7: 2023-02-09 15:51:41.517401: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +4: 2023-02-09 16:21:56.580555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.580586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.580601: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.580634: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.580636: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.580645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.580666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.580718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583738: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583737: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +4: 2023-02-09 16:21:56.583749: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 16:21:56.583751: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 16:21:56.583750: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 16:21:56.583753: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 16:21:56.583753: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 16:21:56.583758: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 16:21:56.583756: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +4: 2023-02-09 16:21:56.583758: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.600800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.600828: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.600840: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.600861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.600865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.600872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.600879: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.600948: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603333: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603340: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603341: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603340: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603341: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603349: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.603354: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.603346: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +5: 2023-02-09 16:21:56.603354: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.603360: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.603359: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.603362: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.603364: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +5: 2023-02-09 16:21:56.603366: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.615389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.615404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.615415: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.615427: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.615430: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.615431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.615555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.615561: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616488: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616505: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616530: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616527: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616541: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.616615: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617840: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617843: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617842: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617845: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617847: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617857: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.617857: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.617855: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +6: 2023-02-09 16:21:56.617866: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.617865: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.617867: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.617868: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.617870: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +6: 2023-02-09 16:21:56.617876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.618978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.619005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.619024: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.619036: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.619049: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.619079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.619131: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.619132: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +3: 2023-02-09 16:21:56.619831: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 16:21:56.619831: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 16:21:56.619832: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 16:21:56.619836: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 16:21:56.619836: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 16:21:56.619838: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 16:21:56.619839: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +3: 2023-02-09 16:21:56.619840: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622789: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622791: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622805: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622805: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622803: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +7: 2023-02-09 16:21:56.622810: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622817: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622818: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622820: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +7: 2023-02-09 16:21:56.622824: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.643822: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.643850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.643872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.643887: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: 2023-02-09 16:21:56.644093: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.643907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.643913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.643932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.644128: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +0: 2023-02-09 16:21:56.644005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.644140: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.644169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.644178: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.644196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.644201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.644256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646681: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646689: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646699: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 16:21:56.646689: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646710: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 16:21:56.646712: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 16:21:56.646693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +1: 2023-02-09 16:21:56.646717: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 16:21:56.646717: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 16:21:56.646718: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 16:21:56.646723: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +1: 2023-02-09 16:21:56.646723: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.650768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.650793: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.650819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.650829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.650838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.650842: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.651028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.651030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125 +2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653221: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653238: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.653237: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.653232: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653232: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653236: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653238: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +2: 2023-02-09 16:21:56.653249: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.653250: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.653253: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.653255: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.653256: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +2: 2023-02-09 16:21:56.653257: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646026: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646041: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646044: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646045: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646049: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646050: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646052: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646053: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: 2023-02-09 16:21:56.646554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro +0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64 +0: 2023-02-09 16:21:56.646568: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +0: Loading extension module scaled_upper_triang_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module scaled_masked_softmax_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module scaled_masked_softmax_cuda... +0: Successfully preprocessed all matching files. +0: Detected CUDA files, patching ldflags +0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +0: Building extension module fused_mix_prec_layer_norm_cuda... +0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +0: Loading extension module fused_mix_prec_layer_norm_cuda... +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +4: warnings.warn( +6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +6: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +5: warnings.warn( +3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +3: warnings.warn( +2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +2: warnings.warn( +7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +7: warnings.warn( +1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +1: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +0: warnings.warn( +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... +1: Building extension module utils... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: +2: +2: +1: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: +3: +3: +3: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: +5: +5: +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: +6: +6: +6: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: +7: +7: +1: Loading extension module utils... +0: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +1: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +0: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +2: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +2: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +4: Loading extension module utils... +2: Loading extension module utils... +5: Loading extension module utils... +3: Loading extension module utils... +5: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +2: Loading extension module utils... +3: Loading extension module utils... +6: Loading extension module utils... +2: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +3: Loading extension module utils... +6: Loading extension module utils... +6: Loading extension module utils... +5: Loading extension module utils... +6: Loading extension module utils... +5: Loading extension module utils... +6: Loading extension module utils... +5: Loading extension module utils... +3: Loading extension module utils... +5: Loading extension module utils... +5: Loading extension module utils... +7: Loading extension module utils... +5: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +3: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +7: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +0: +0: +0: +0: Loading extension module utils...Loading extension module utils... +0: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +0: +0: +0: +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: +0: Loading extension module utils...Loading extension module utils... +0: +0: Loading extension module utils... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +1: +1: Loading extension module utils...Loading extension module utils... +1: +1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +1: No modifications detected for re-loaded extension module utils, skipping build step... +1: Loading extension module utils... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: +4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +4: +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +4: No modifications detected for re-loaded extension module utils, skipping build step... +4: Loading extension module utils... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +6: +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +6: No modifications detected for re-loaded extension module utils, skipping build step... +6: Loading extension module utils... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +5: +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +5: +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +5: No modifications detected for re-loaded extension module utils, skipping build step... +5: Loading extension module utils... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +3: +3: +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +3: +3: Loading extension module utils... +3: No modifications detected for re-loaded extension module utils, skipping build step... +3: Loading extension module utils... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +2: +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +2: +2: Loading extension module utils... +2: Loading extension module utils... +2: No modifications detected for re-loaded extension module utils, skipping build step... +2: Loading extension module utils... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: +7: +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +7: +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +7: No modifications detected for re-loaded extension module utils, skipping build step... +7: Loading extension module utils... +0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +0: No modifications detected for re-loaded extension module utils, skipping build step... +0: Loading extension module utils... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/619m22b4b8/logs/2817194.out b/619m22b4b8/logs/2817194.out new file mode 100644 index 0000000000000000000000000000000000000000..f18eba5f7abedefb6de77b92646c2b94da5d258d --- /dev/null +++ b/619m22b4b8/logs/2817194.out @@ -0,0 +1,6681 @@ +Model parameters: d_model 1536 ffw_size 6144 kv_size 128 n_heads 12 n_layers 19 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 19 --hidden-size 1536 --num-attention-heads 12 --kv-channels 128 --ffn-hidden-size 6144 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-619m22b4b8val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --no-load-optim --reset-progress --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --tensorboard-dir tensorboard_619m22b4b8val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_619m22b4b8 --load checkpoints_619m22b4b8 --train-weighted-split-paths-path train4b8.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/2817194.json --zero-stage 0 +START 2817194: Thu 09 Feb 2023 03:37:00 PM EET +0: +0: +0: ======================= ROCm System Management Interface ======================= +0: ================================= Concise Info ================================= +0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +0: 0 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 2 38.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 4 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: 6 40.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +0: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +0: ================================================================================ +0: ============================= End of ROCm SMI Log ============================== +4: +4: +4: ======================= ROCm System Management Interface ======================= +4: ================================= Concise Info ================================= +4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +4: 0 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 2 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 4 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: 6 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +4: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +4: ================================================================================ +4: ============================= End of ROCm SMI Log ============================== +1: +1: +1: ======================= ROCm System Management Interface ======================= +1: ================================= Concise Info ================================= +1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +1: 0 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 2 41.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 4 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: 6 46.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +1: 7 37.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +1: ================================================================================ +1: ============================= End of ROCm SMI Log ============================== +2: +2: +2: ======================= ROCm System Management Interface ======================= +2: ================================= Concise Info ================================= +2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +2: 0 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 1 53.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 2 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 4 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: 6 39.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +2: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +2: ================================================================================ +2: ============================= End of ROCm SMI Log ============================== +6: +6: +6: ======================= ROCm System Management Interface ======================= +6: ================================= Concise Info ================================= +6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +6: 0 49.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 2 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 4 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: 6 38.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +6: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +6: ================================================================================ +6: ============================= End of ROCm SMI Log ============================== +3: +3: +3: ======================= ROCm System Management Interface ======================= +3: ================================= Concise Info ================================= +3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +3: 0 45.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 2 35.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 4 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: 6 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +3: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +3: ================================================================================ +3: ============================= End of ROCm SMI Log ============================== +7: +7: +7: ======================= ROCm System Management Interface ======================= +7: ================================= Concise Info ================================= +7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +7: 0 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 2 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 4 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: 6 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +7: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +7: ================================================================================ +7: ============================= End of ROCm SMI Log ============================== +5: +5: +5: ======================= ROCm System Management Interface ======================= +5: ================================= Concise Info ================================= +5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +5: 0 47.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 2 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 4 48.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: 6 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +5: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +5: ================================================================================ +5: ============================= End of ROCm SMI Log ============================== +7: Launching on nid006946 (7/8), master nid005652 port 9999, GPUs 8, CUDA: True +0: Launching on nid005652 (0/8), master nid005652 port 9999, GPUs 8, CUDA: True +1: Launching on nid005872 (1/8), master nid005652 port 9999, GPUs 8, CUDA: True +3: Launching on nid005888 (3/8), master nid005652 port 9999, GPUs 8, CUDA: True +2: Launching on nid005873 (2/8), master nid005652 port 9999, GPUs 8, CUDA: True +5: Launching on nid006348 (5/8), master nid005652 port 9999, GPUs 8, CUDA: True +4: Launching on nid005948 (4/8), master nid005652 port 9999, GPUs 8, CUDA: True +6: Launching on nid006945 (6/8), master nid005652 port 9999, GPUs 8, CUDA: True +0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. +0: using torch.bfloat16 for parameters ... +0: ------------------------ arguments ------------------------ +0: abort_on_unmet_fused_kernel_constraints ......... False +0: accumulate_allreduce_grads_in_fp32 .............. True +0: adam_beta1 ...................................... 0.9 +0: adam_beta2 ...................................... 0.999 +0: adam_eps ........................................ 1e-08 +0: adlr_autoresume ................................. False +0: adlr_autoresume_interval ........................ 1000 +0: apply_query_key_layer_scaling ................... True +0: apply_residual_connection_post_layernorm ........ False +0: attention_dropout ............................... 0.1 +0: attention_softmax_in_fp32 ....................... False +0: bert_binary_head ................................ True +0: bert_load ....................................... None +0: bf16 ............................................ True +0: bias_dropout_fusion ............................. True +0: bias_gelu_fusion ................................ True +0: biencoder_projection_dim ........................ 0 +0: biencoder_shared_query_context_model ............ False +0: block_data_path ................................. None +0: checkpoint_activations .......................... False +0: checkpoint_in_cpu ............................... False +0: checkpoint_num_layers ........................... 1 +0: clip_grad ....................................... 1.0 +0: codecarbon_dir .................................. None +0: consumed_train_samples .......................... 0 +0: consumed_train_tokens ........................... 0 +0: consumed_valid_samples .......................... 0 +0: contigious_checkpointing ........................ False +0: cpu_optimizer ................................... False +0: cpu_torch_adam .................................. False +0: curriculum_learning ............................. False +0: data_impl ....................................... mmap +0: data_parallel_size .............................. 64 +0: data_path ....................................... None +0: dataloader_type ................................. single +0: DDP_impl ........................................ local +0: decoder_seq_length .............................. None +0: deepscale ....................................... False +0: deepscale_config ................................ None +0: deepspeed ....................................... True +0: deepspeed_activation_checkpointing .............. False +0: deepspeed_config ................................ ds_configs/2817194.json +0: deepspeed_mpi ................................... False +0: distribute_checkpointed_activations ............. False +0: distributed_backend ............................. nccl +0: embed_layernorm ................................. False +0: embedding_path .................................. None +0: encoder_seq_length .............................. 2048 +0: eod_mask_loss ................................... False +0: eval_interval ................................... 1 +0: eval_iters ...................................... 100 +0: eval_only ....................................... None +0: evidence_data_path .............................. None +0: exit_duration_in_mins ........................... None +0: exit_interval ................................... None +0: ffn_hidden_size ................................. 6144 +0: finetune ........................................ False +0: fp16 ............................................ False +0: fp16_lm_cross_entropy ........................... False +0: fp32_residual_connection ........................ False +0: gigaflos_no_embeds .............................. 0 +0: global_batch_size ............................... 256 +0: glu_activation .................................. None +0: hidden_dropout .................................. 0.1 +0: hidden_size ..................................... 1536 +0: hysteresis ...................................... 2 +0: ict_head_size ................................... None +0: ict_load ........................................ None +0: img_dim ......................................... 224 +0: indexer_batch_size .............................. 128 +0: indexer_log_interval ............................ 1000 +0: inference ....................................... False +0: init_method_std ................................. 0.02 +0: init_method_xavier_uniform ...................... False +0: initial_loss_scale .............................. 4294967296 +0: kill_switch_path ................................ kill-switch-619m22b4b8val +0: kv_channels ..................................... 128 +0: layer_norm_fusion ............................... True +0: layernorm_epsilon ............................... 1e-05 +0: lazy_mpu_init ................................... None +0: load ............................................ checkpoints_619m22b4b8 +0: local_rank ...................................... None +0: log_batch_size_to_tensorboard ................... True +0: log_interval .................................... 10 +0: log_learning_rate_to_tensorboard ................ True +0: log_level ....................................... None +0: log_level_replica ............................... None +0: log_loss_scale_to_tensorboard ................... True +0: log_num_zeros_in_grad ........................... False +0: log_params_norm ................................. False +0: log_path ........................................ None +0: log_timers_to_tensorboard ....................... True +0: log_validation_ppl_to_tensorboard ............... True +0: loss_on_targets_only ............................ False +0: loss_scale ...................................... None +0: loss_scale_window ............................... 1000 +0: lr .............................................. 0.0002 +0: lr_decay_iters .................................. None +0: lr_decay_samples ................................ 1 +0: lr_decay_style .................................. cosine +0: lr_decay_tokens ................................. None +0: lr_warmup_fraction .............................. None +0: lr_warmup_iters ................................. 0 +0: lr_warmup_samples ............................... 0 +0: make_vocab_size_divisible_by .................... 128 +0: mask_prob ....................................... 0.15 +0: masked_softmax_fusion ........................... True +0: max_position_embeddings ......................... 2048 +0: mean_noise_span_length .......................... None +0: memory_centric_tiled_linear ..................... False +0: merge_file ...................................... gpt2/merges.txt +0: micro_batch_size ................................ 4 +0: min_loss_scale .................................. 1.0 +0: min_lr .......................................... 2e-05 +0: mmap_warmup ..................................... False +0: no_load_optim ................................... True +0: no_load_rng ..................................... None +0: no_save_optim ................................... None +0: no_save_rng ..................................... None +0: noise_density ................................... None +0: num_attention_heads ............................. 12 +0: num_channels .................................... 3 +0: num_classes ..................................... 1000 +0: num_layers ...................................... 19 +0: num_layers_per_virtual_pipeline_stage ........... None +0: num_workers ..................................... 2 +0: onnx_safe ....................................... None +0: openai_gelu ..................................... False +0: optimizer ....................................... adam +0: optimizer_fusion ................................ True +0: override_lr_scheduler ........................... True +0: pad_vocab_size_to ............................... None +0: params_dtype .................................... torch.bfloat16 +0: partition_activations ........................... False +0: patch_dim ....................................... 16 +0: pipeline_model_parallel_size .................... 1 +0: position_embedding_type ......................... PositionEmbeddingType.absolute +0: pp_partition_method ............................. None +0: profile_backward ................................ False +0: query_in_block_prob ............................. 0.1 +0: rampup_batch_size ............................... None +0: rank ............................................ 0 +0: remote_device ................................... none +0: reset_attention_mask ............................ False +0: reset_position_ids .............................. False +0: reset_progress .................................. True +0: retriever_report_topk_accuracies ................ [] +0: retriever_score_scaling ......................... False +0: retriever_seq_length ............................ 256 +0: reweight_loss_based_on_position_frequency ....... False +0: sample_rate ..................................... 1.0 +0: save ............................................ checkpoints_619m22b4b8 +0: save_interval ................................... 1000 +0: scatter_gather_tensors_in_pipeline .............. True +0: scattered_embeddings ............................ False +0: seed ............................................ 1234 +0: seq_length ...................................... 2048 +0: sgd_momentum .................................... 0.9 +0: short_seq_prob .................................. 0.1 +0: skip_train_iteration_range ...................... None +0: split ........................................... None +0: split_transformers .............................. False +0: sync_tp_duplicated_parameters ................... False +0: synchronize_each_layer .......................... False +0: tensor_model_parallel_size ...................... 1 +0: tensorboard_dir ................................. tensorboard_619m22b4b8val +0: tensorboard_log_interval ........................ 1 +0: tensorboard_queue_size .......................... 5 +0: test_weighted_split_paths ....................... None +0: test_weighted_split_paths_path .................. None +0: tile_factor ..................................... 1 +0: titles_data_path ................................ None +0: tokenizer_name_or_path .......................... None +0: tokenizer_type .................................. GPT2BPETokenizer +0: train_iters ..................................... None +0: train_samples ................................... 1 +0: train_tokens .................................... None +0: train_weighted_split_names ...................... ['train'] +0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document']] +0: train_weighted_split_paths_path ................. None +0: train_weighted_split_splits ..................... [['0:1']] +0: train_weighted_split_weights .................... [['1.0']] +0: universal_checkpoint ............................ False +0: use_bnb_optimizer ............................... False +0: use_checkpoint_lr_scheduler ..................... False +0: use_contiguous_buffers_in_ddp ................... True +0: use_cpu_initialization .......................... None +0: use_one_sent_docs ............................... False +0: use_pin_memory .................................. False +0: valid_num_workers ............................... 2 +0: valid_weighted_split_names ...................... ['validation'] +0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] +0: valid_weighted_split_paths_path ................. None +0: valid_weighted_split_splits ..................... [['0:1']] +0: valid_weighted_split_weights .................... [['1.0']] +0: virtual_pipeline_model_parallel_size ............ None +0: vocab_extra_ids ................................. 0 +0: vocab_file ...................................... gpt2/vocab.json +0: weight_decay .................................... 0.1 +0: world_size ...................................... 64 +0: zero_allgather_bucket_size ...................... 0.0 +0: zero_contigious_gradients ....................... False +0: zero_reduce_bucket_size ......................... 0.0 +0: zero_reduce_scatter ............................. False +0: zero_stage ...................................... 0 +0: -------------------- end of arguments --------------------- +0: setting number of micro-batches to constant 1 +0: > building GPT2BPETokenizer tokenizer ... +0: > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +0: DeepSpeed general environment info: +0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] +0: torch version .................... 1.13.0+rocm5.2 +0: torch cuda version ............... None +0: torch hip version ................ 5.2.21151-afdc89f8 +0: nvcc version ..................... None +0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] +0: deepspeed info ................... 0.7.5, unknown, unknown +0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 +0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** +0: > initializing torch distributed ... +0: [2023-02-09 17:17:32,764] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +7: > setting tensorboard ... +0: > initializing tensor model parallel with size 1 +0: > initializing pipeline model parallel with size 1 +0: > setting random seeds to 1234 ... +0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 +0: > compiling dataset index builder ... +0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: make: Nothing to be done for 'default'. +0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' +0: >>> done with dataset index builder. Compilation time: 0.160 seconds +0: > compiling and loading fused kernels ... +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 102 +0: [1/1] c++ scaled_masked_softmax_hip.cuda.o scaled_masked_softmax_hip.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_masked_softmax_cuda.so +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] +0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] +0: Total number of unsupported CUDA function calls: 0 +0: +0: +0: Total number of replaced kernel launches: 67 +0: [1/1] c++ layer_norm_cuda.o layer_norm_hip_kernel.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so +0: >>> done with compiling and loading fused kernels. Compilation time: 40.954 seconds +0: time to initialize megatron (seconds): 61.158 +0: [after megatron is initialized] datetime: 2023-02-09 17:18:21 +0: building GPT model ... +0: [2023-02-09 17:18:21,312] [INFO] [utils.py:827:see_memory_usage] Before Building Model +0: [2023-02-09 17:18:21,313] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +0: [2023-02-09 17:18:21,313] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.07 GB, percent = 6.0% +0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi +0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4 +0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63} +0: [2023-02-09 17:18:23,295] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +0: stage=0 layers=26 +0: 0: _to_float16 +0: 1: EmbeddingPipe +0: 2: +0: 3: ParallelTransformerLayerPipe +0: 4: ParallelTransformerLayerPipe +0: 5: ParallelTransformerLayerPipe +0: 6: ParallelTransformerLayerPipe +0: 7: ParallelTransformerLayerPipe +0: 8: ParallelTransformerLayerPipe +0: 9: ParallelTransformerLayerPipe +0: 10: ParallelTransformerLayerPipe +0: 11: ParallelTransformerLayerPipe +0: 12: ParallelTransformerLayerPipe +0: 13: ParallelTransformerLayerPipe +0: 14: ParallelTransformerLayerPipe +0: 15: ParallelTransformerLayerPipe +0: 16: ParallelTransformerLayerPipe +0: 17: ParallelTransformerLayerPipe +0: 18: ParallelTransformerLayerPipe +0: 19: ParallelTransformerLayerPipe +0: 20: ParallelTransformerLayerPipe +0: 21: ParallelTransformerLayerPipe +0: 22: undo +0: 23: MixedFusedLayerNorm +0: 24: EmbeddingPipe +0: 25: float16_to_fp32 +0: loss: CrossEntropy +0: [2023-02-09 17:18:23,629] [INFO] [utils.py:827:see_memory_usage] After Building Model +0: [2023-02-09 17:18:23,629] [INFO] [utils.py:828:see_memory_usage] MA 1.16 GB Max_MA 1.16 GB CA 1.2 GB Max_CA 1 GB +0: [2023-02-09 17:18:23,630] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.1 GB, percent = 6.0% +0: setting training iterations to 0 +0: > learning rate decay style: cosine +0: DeepSpeed is enabled. +0: [2023-02-09 17:18:23,632] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown +0: [2023-02-09 17:18:37,195] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +0: [2023-02-09 17:18:37,196] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +0: [2023-02-09 17:18:37,196] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +0: [2023-02-09 17:18:37,203] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam +0: [2023-02-09 17:18:37,203] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer +0: [2023-02-09 17:18:37,320] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer +0: [2023-02-09 17:18:37,320] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.17 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-09 17:18:37,321] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.79 GB, percent = 6.1% +1: ninja: no work to do. +0: Time to load utils op: 0.10962080955505371 seconds +1: Time to load utils op: 0.21018242835998535 seconds +1: Time to load utils op: 0.20357799530029297 seconds +1: Time to load utils op: 0.20396184921264648 seconds +1: Time to load utils op: 0.2048628330230713 seconds +1: Time to load utils op: 0.20533394813537598 seconds +1: Time to load utils op: 0.10258698463439941 seconds +1: Time to load utils op: 0.10262513160705566 seconds +1: Time to load utils op: 0.10325193405151367 seconds +0: Time to load utils op: 0.21004676818847656 seconds +0: Time to load utils op: 0.21013331413269043 seconds +0: Time to load utils op: 0.21013808250427246 seconds +0: Time to load utils op: 0.2106330394744873 seconds +0: Time to load utils op: 0.21227288246154785 seconds +0: Time to load utils op: 0.21087217330932617 seconds +0: Time to load utils op: 0.21096086502075195 seconds +3: Time to load utils op: 0.1182107925415039 seconds +3: Time to load utils op: 0.11820793151855469 seconds +3: Time to load utils op: 0.11823678016662598 seconds +3: Time to load utils op: 0.1182565689086914 seconds +5: Time to load utils op: 0.11755561828613281 seconds +3: Time to load utils op: 0.11827826499938965 secondsTime to load utils op: 0.11826634407043457 seconds +3: Time to load utils op: 0.11828136444091797 seconds +3: +3: Time to load utils op: 0.11828327178955078 seconds +5: Time to load utils op: 0.11757493019104004 seconds +5: Time to load utils op: 0.11757326126098633 seconds +5: Time to load utils op: 0.11759710311889648 secondsTime to load utils op: 0.11760687828063965 seconds +5: Time to load utils op: 0.11760067939758301 seconds +5: +5: Time to load utils op: 0.11760997772216797 seconds +5: Time to load utils op: 0.11764192581176758 seconds +4: Time to load utils op: 0.11870861053466797 secondsTime to load utils op: 0.11871576309204102 seconds +4: +4: Time to load utils op: 0.11873030662536621 seconds +4: Time to load utils op: 0.11875581741333008 seconds +4: Time to load utils op: 0.1187753677368164 seconds +4: Time to load utils op: 0.11878371238708496 seconds +4: Time to load utils op: 0.11880040168762207 seconds +4: Time to load utils op: 0.11878705024719238 seconds +6: Time to load utils op: 0.11669206619262695 seconds +6: Time to load utils op: 0.1167149543762207 seconds +6: Time to load utils op: 0.11674094200134277 seconds +6: Time to load utils op: 0.11674809455871582 seconds +6: Time to load utils op: 0.11675596237182617 seconds +6: Time to load utils op: 0.11675882339477539 seconds +6: Time to load utils op: 0.11674857139587402 secondsTime to load utils op: 0.1167762279510498 seconds +6: +7: Time to load utils op: 0.11684489250183105 seconds +7: Time to load utils op: 0.11685681343078613 seconds +7: Time to load utils op: 0.11686205863952637 secondsTime to load utils op: 0.11686086654663086 seconds +7: Time to load utils op: 0.11696910858154297 seconds +7: +7: Time to load utils op: 0.11687970161437988 secondsTime to load utils op: 0.11687183380126953 seconds +7: +7: Time to load utils op: 0.11689138412475586 seconds +2: Time to load utils op: 0.12786459922790527 seconds +2: Time to load utils op: 0.12790203094482422 seconds +2: Time to load utils op: 0.12788891792297363 secondsTime to load utils op: 0.1278841495513916 seconds +2: +2: Time to load utils op: 0.12795495986938477 seconds +2: Time to load utils op: 0.1279618740081787 seconds +2: Time to load utils op: 0.12790703773498535 seconds +2: Time to load utils op: 0.12790751457214355 seconds +0: [2023-02-09 17:18:37,537] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 +0: [2023-02-09 17:18:37,537] [INFO] [utils.py:828:see_memory_usage] MA 1.15 GB Max_MA 1.15 GB CA 1.22 GB Max_CA 1 GB +0: [2023-02-09 17:18:37,538] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.8 GB, percent = 6.1% +0: Time to load utils op: 0.0007245540618896484 seconds +0: Time to load utils op: 0.0006594657897949219 secondsTime to load utils op: 0.0007064342498779297 secondsTime to load utils op: 0.0007054805755615234 seconds +0: +0: Time to load utils op: 0.0006959438323974609 seconds +0: +0: Time to load utils op: 0.0007164478302001953 secondsTime to load utils op: 0.0007109642028808594 seconds +0: +1: Time to load utils op: 0.0005261898040771484 seconds +1: Time to load utils op: 0.00054931640625 seconds +1: Time to load utils op: 0.0004277229309082031 seconds +1: Time to load utils op: 0.0005567073822021484 seconds +1: Time to load utils op: 0.0005435943603515625 seconds +1: Time to load utils op: 0.0006091594696044922 seconds +1: Time to load utils op: 0.0005998611450195312 seconds +1: Time to load utils op: 0.0004904270172119141 seconds +4: Time to load utils op: 0.0008001327514648438 seconds +4: Time to load utils op: 0.0011832714080810547 seconds +4: Time to load utils op: 0.0011816024780273438 seconds +4: Time to load utils op: 0.0011706352233886719 seconds +4: Time to load utils op: 0.0011458396911621094 seconds +4: Time to load utils op: 0.0012035369873046875 seconds +4: Time to load utils op: 0.0011818408966064453 seconds +4: Time to load utils op: 0.0012164115905761719 seconds +6: Time to load utils op: 0.001087188720703125 seconds +6: Time to load utils op: 0.0013973712921142578 seconds +6: Time to load utils op: 0.0013020038604736328 seconds +6: Time to load utils op: 0.0013954639434814453 seconds +6: Time to load utils op: 0.00141143798828125 seconds +6: Time to load utils op: 0.0014460086822509766 seconds +6: Time to load utils op: 0.0014007091522216797 seconds +6: Time to load utils op: 0.0013928413391113281 seconds +5: Time to load utils op: 0.0012030601501464844 seconds +5: Time to load utils op: 0.0013544559478759766 seconds +5: Time to load utils op: 0.0016703605651855469 secondsTime to load utils op: 0.0016498565673828125 seconds +5: +5: Time to load utils op: 0.0016965866088867188 seconds +5: Time to load utils op: 0.0016121864318847656 seconds +5: Time to load utils op: 0.0016260147094726562 seconds +5: Time to load utils op: 0.001665353775024414 seconds +3: Time to load utils op: 0.0010631084442138672 seconds +3: Time to load utils op: 0.0012454986572265625 seconds +3: Time to load utils op: 0.001399993896484375 seconds +3: Time to load utils op: 0.0013518333435058594 seconds +3: Time to load utils op: 0.0013680458068847656 seconds +3: Time to load utils op: 0.0013763904571533203 secondsTime to load utils op: 0.0014715194702148438 seconds +3: +3: Time to load utils op: 0.0014138221740722656 seconds +2: Time to load utils op: 0.0012407302856445312 seconds +2: Time to load utils op: 0.0012636184692382812 seconds +2: Time to load utils op: 0.0014498233795166016 seconds +2: Time to load utils op: 0.001447916030883789 seconds +2: Time to load utils op: 0.001499176025390625 seconds +2: Time to load utils op: 0.001567840576171875 seconds +2: Time to load utils op: 0.0015170574188232422 seconds +2: Time to load utils op: 0.0015838146209716797 seconds +0: [2023-02-09 17:18:37,653] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 +0: [2023-02-09 17:18:37,654] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 17:18:37,654] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +7: Time to load utils op: 0.0006716251373291016 seconds +7: Time to load utils op: 0.0007715225219726562 seconds +7: Time to load utils op: 0.0007648468017578125 seconds +7: Time to load utils op: 0.000804901123046875 seconds +7: Time to load utils op: 0.0010006427764892578 seconds +7: Time to load utils op: 0.0010712146759033203 seconds +7: Time to load utils op: 0.0009238719940185547 seconds +7: Time to load utils op: 0.0010161399841308594 seconds +0: [2023-02-09 17:18:37,757] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 +0: [2023-02-09 17:18:37,758] [INFO] [utils.py:828:see_memory_usage] MA 2.45 GB Max_MA 2.45 GB CA 3.13 GB Max_CA 3 GB +0: [2023-02-09 17:18:37,758] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +0: [2023-02-09 17:18:37,862] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 +0: [2023-02-09 17:18:37,862] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:37,863] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +0: [2023-02-09 17:18:37,963] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 +0: [2023-02-09 17:18:37,963] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:37,964] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +0: [2023-02-09 17:18:38,069] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 +0: [2023-02-09 17:18:38,069] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:38,069] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +0: [2023-02-09 17:18:38,172] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer +0: [2023-02-09 17:18:38,172] [INFO] [utils.py:828:see_memory_usage] MA 3.49 GB Max_MA 3.49 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:38,172] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +0: [2023-02-09 17:18:38,280] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer +0: [2023-02-09 17:18:38,280] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:38,281] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +0: [2023-02-09 17:18:38,382] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer +0: [2023-02-09 17:18:38,382] [INFO] [utils.py:828:see_memory_usage] MA 3.57 GB Max_MA 3.57 GB CA 4.68 GB Max_CA 5 GB +0: [2023-02-09 17:18:38,382] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 30.95 GB, percent = 6.1% +0: [2023-02-09 17:18:38,382] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +0: [2023-02-09 17:18:38,383] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +0: [2023-02-09 17:18:38,383] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +0: [2023-02-09 17:18:38,383] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] +0: [2023-02-09 17:18:38,383] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: +0: [2023-02-09 17:18:38,383] [INFO] [config.py:1011:print] activation_checkpointing_config { +0: "partition_activations": false, +0: "contiguous_memory_optimization": false, +0: "cpu_checkpointing": false, +0: "number_checkpoints": null, +0: "synchronize_checkpoint_boundary": false, +0: "profile": false +0: } +0: [2023-02-09 17:18:38,383] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +0: [2023-02-09 17:18:38,383] [INFO] [config.py:1011:print] amp_enabled .................. False +0: [2023-02-09 17:18:38,383] [INFO] [config.py:1011:print] amp_params ................... False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] autotuning_config ............ { +0: "enabled": false, +0: "start_step": null, +0: "end_step": null, +0: "metric_path": null, +0: "arg_mappings": null, +0: "metric": "throughput", +0: "model_info": null, +0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", +0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", +0: "overwrite": true, +0: "fast": true, +0: "start_profile_step": 3, +0: "end_profile_step": 5, +0: "tuner_type": "gridsearch", +0: "tuner_early_stopping": 5, +0: "tuner_num_trials": 50, +0: "model_info_path": null, +0: "mp_size": 1, +0: "max_train_batch_size": null, +0: "min_train_batch_size": 1, +0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +0: "min_train_micro_batch_size_per_gpu": 1, +0: "num_tuning_micro_batch_sizes": 3 +0: } +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] bfloat16_enabled ............. True +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] comms_config ................. +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] communication_data_type ...... None +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa +0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] curriculum_enabled ........... False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] curriculum_params ............ False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] dataloader_drop_last ......... False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] disable_allgather ............ False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] dump_state ................... False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] elasticity_enabled ........... False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] flops_profiler_config ........ { +0: "enabled": false, +0: "profile_step": 1, +0: "module_depth": -1, +0: "top_modules": 1, +0: "detailed": true, +0: "output_file": null +0: } +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] fp16_auto_cast ............... None +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] fp16_enabled ................. False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] global_rank .................. 0 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 1 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 +0: [2023-02-09 17:18:38,384] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] load_universal_checkpoint .... False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] loss_scale ................... 1.0 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] memory_breakdown ............. False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] monitor_config ............... +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] nebula_config ................ { +0: "enabled": false, +0: "persistent_storage_path": null, +0: "persistent_time_interval": 100, +0: "num_of_version_in_retention": 2, +0: "enable_nebula_load": true, +0: "load_path": null +0: } +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] optimizer_name ............... None +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] optimizer_params ............. None +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] pld_enabled .................. False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] pld_params ................... False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] prescale_gradients ........... False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] scheduler_name ............... None +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] scheduler_params ............. None +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] sparse_attention ............. None +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] steps_per_print .............. 2000 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] train_batch_size ............. 256 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 4 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] use_node_local_storage ....... False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] world_size ................... 64 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] zero_enabled ................. False +0: [2023-02-09 17:18:38,385] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 +0: [2023-02-09 17:18:38,385] [INFO] [config.py:996:print_user_config] json = { +0: "train_micro_batch_size_per_gpu": 4, +0: "train_batch_size": 256, +0: "gradient_clipping": 1.0, +0: "zero_optimization": { +0: "stage": 0 +0: }, +0: "bf16": { +0: "enabled": true +0: }, +0: "steps_per_print": 2.000000e+03, +0: "wall_clock_breakdown": false +0: } +0: Time to load utils op: 0.0004162788391113281 seconds +0: [2023-02-09 17:18:38,386] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4 +0: [2023-02-09 17:18:38,396] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=26 [0, 26) STAGE_PARAMS=618714624 (618.715M) TOTAL_PARAMS=618714624 (618.715M) UNIQUE_PARAMS=618714624 (618.715M) +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +5: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +7: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +6: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +2: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +3: [2023-02-09 17:18:38,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt... +0: [2023-02-09 17:18:38,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +0: [2023-02-09 17:18:38,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +1: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +2: [2023-02-09 17:18:38,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +6: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +4: [2023-02-09 17:18:38,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/mp_rank_00_model_states.pt. +5: [2023-02-09 17:18:38,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +5: [2023-02-09 17:18:38,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +1: [2023-02-09 17:18:38,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +0: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +2: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +3: [2023-02-09 17:18:38,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +6: [2023-02-09 17:18:38,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +7: [2023-02-09 17:18:38,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt... +4: [2023-02-09 17:18:38,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:38,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:38,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:38,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:38,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:38,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +7: [2023-02-09 17:18:38,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:38,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:38,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +1: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +2: [2023-02-09 17:18:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +5: [2023-02-09 17:18:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:38,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:38,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:38,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:38,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:38,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +6: [2023-02-09 17:18:38,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +4: [2023-02-09 17:18:38,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:38,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:38,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:38,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:38,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +0: [2023-02-09 17:18:38,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_01-model_00-model_states.pt. +3: [2023-02-09 17:18:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:38,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:38,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:38,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:38,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:38,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:38,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:38,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +2: [2023-02-09 17:18:39,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:39,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +5: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +7: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +6: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +1: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +3: [2023-02-09 17:18:39,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +0: [2023-02-09 17:18:39,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt... +4: [2023-02-09 17:18:39,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +4: [2023-02-09 17:18:39,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +6: [2023-02-09 17:18:39,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +7: [2023-02-09 17:18:39,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +3: [2023-02-09 17:18:39,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +1: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +2: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +0: [2023-02-09 17:18:39,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_03-model_00-model_states.pt. +5: [2023-02-09 17:18:39,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +2: [2023-02-09 17:18:39,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +7: [2023-02-09 17:18:39,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +1: [2023-02-09 17:18:39,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +3: [2023-02-09 17:18:39,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +6: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +4: [2023-02-09 17:18:39,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +0: [2023-02-09 17:18:39,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt... +5: [2023-02-09 17:18:39,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +7: [2023-02-09 17:18:39,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +5: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +3: [2023-02-09 17:18:39,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +1: [2023-02-09 17:18:39,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +2: [2023-02-09 17:18:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +6: [2023-02-09 17:18:39,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +4: [2023-02-09 17:18:39,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_04-model_00-model_states.pt. +0: [2023-02-09 17:18:39,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +2: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +5: [2023-02-09 17:18:39,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +3: [2023-02-09 17:18:39,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +4: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +1: [2023-02-09 17:18:39,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +2: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +7: [2023-02-09 17:18:39,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +1: [2023-02-09 17:18:39,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +5: [2023-02-09 17:18:39,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +0: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt... +6: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +3: [2023-02-09 17:18:39,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +7: [2023-02-09 17:18:39,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +4: [2023-02-09 17:18:39,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +6: [2023-02-09 17:18:39,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_05-model_00-model_states.pt. +0: [2023-02-09 17:18:39,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +1: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +0: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +5: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +6: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +7: [2023-02-09 17:18:39,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +4: [2023-02-09 17:18:39,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +3: [2023-02-09 17:18:39,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt... +2: [2023-02-09 17:18:39,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +2: [2023-02-09 17:18:39,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +5: [2023-02-09 17:18:39,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +7: [2023-02-09 17:18:39,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +1: [2023-02-09 17:18:39,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +3: [2023-02-09 17:18:39,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +4: [2023-02-09 17:18:39,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +6: [2023-02-09 17:18:39,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_06-model_00-model_states.pt. +0: [2023-02-09 17:18:39,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:39,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:39,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +5: [2023-02-09 17:18:39,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:39,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:39,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +5: [2023-02-09 17:18:39,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:39,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +3: [2023-02-09 17:18:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +3: [2023-02-09 17:18:39,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:39,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +7: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +6: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +1: [2023-02-09 17:18:39,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +2: [2023-02-09 17:18:39,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +0: [2023-02-09 17:18:39,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt... +4: [2023-02-09 17:18:39,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:39,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:39,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +6: [2023-02-09 17:18:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +4: [2023-02-09 17:18:39,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +7: [2023-02-09 17:18:39,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:39,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:39,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +1: [2023-02-09 17:18:39,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:39,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +2: [2023-02-09 17:18:39,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:39,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_07-model_00-model_states.pt. +0: [2023-02-09 17:18:39,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:39,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:40,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:40,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +3: [2023-02-09 17:18:40,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +3: [2023-02-09 17:18:40,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:40,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +5: [2023-02-09 17:18:40,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +5: [2023-02-09 17:18:40,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:40,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +6: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:40,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +2: [2023-02-09 17:18:40,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +1: [2023-02-09 17:18:40,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +4: [2023-02-09 17:18:40,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:40,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:40,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:40,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:40,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:40,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +0: [2023-02-09 17:18:40,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt... +7: [2023-02-09 17:18:40,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +4: [2023-02-09 17:18:40,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +7: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +2: [2023-02-09 17:18:40,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +1: [2023-02-09 17:18:40,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +6: [2023-02-09 17:18:40,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_08-model_00-model_states.pt. +0: [2023-02-09 17:18:40,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +5: [2023-02-09 17:18:40,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +4: [2023-02-09 17:18:40,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +5: [2023-02-09 17:18:40,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +4: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +1: [2023-02-09 17:18:40,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +3: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +7: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +3: [2023-02-09 17:18:40,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +6: [2023-02-09 17:18:40,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +0: [2023-02-09 17:18:40,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt... +2: [2023-02-09 17:18:40,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +1: [2023-02-09 17:18:40,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +6: [2023-02-09 17:18:40,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +7: [2023-02-09 17:18:40,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +2: [2023-02-09 17:18:40,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_09-model_00-model_states.pt. +0: [2023-02-09 17:18:40,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +3: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +5: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +0: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +4: [2023-02-09 17:18:40,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +1: [2023-02-09 17:18:40,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +6: [2023-02-09 17:18:40,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +2: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt... +7: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +7: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +3: [2023-02-09 17:18:40,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +1: [2023-02-09 17:18:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +4: [2023-02-09 17:18:40,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +5: [2023-02-09 17:18:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +0: [2023-02-09 17:18:40,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +6: [2023-02-09 17:18:40,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_10-model_00-model_states.pt. +2: [2023-02-09 17:18:40,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +6: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +3: [2023-02-09 17:18:40,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +7: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +2: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +4: [2023-02-09 17:18:40,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +1: [2023-02-09 17:18:40,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +0: [2023-02-09 17:18:40,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt... +5: [2023-02-09 17:18:40,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +1: [2023-02-09 17:18:40,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +5: [2023-02-09 17:18:40,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +3: [2023-02-09 17:18:40,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +6: [2023-02-09 17:18:40,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +7: [2023-02-09 17:18:40,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +2: [2023-02-09 17:18:40,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +4: [2023-02-09 17:18:40,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_11-model_00-model_states.pt. +0: [2023-02-09 17:18:40,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +2: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +1: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +6: [2023-02-09 17:18:40,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +5: [2023-02-09 17:18:40,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +3: [2023-02-09 17:18:40,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +4: [2023-02-09 17:18:40,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +0: [2023-02-09 17:18:40,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt... +7: [2023-02-09 17:18:40,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +5: [2023-02-09 17:18:40,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +7: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +1: [2023-02-09 17:18:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +2: [2023-02-09 17:18:40,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +4: [2023-02-09 17:18:40,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +3: [2023-02-09 17:18:40,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +6: [2023-02-09 17:18:40,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_12-model_00-model_states.pt. +0: [2023-02-09 17:18:40,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +4: [2023-02-09 17:18:40,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +6: [2023-02-09 17:18:40,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:40,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:40,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +2: [2023-02-09 17:18:40,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +7: [2023-02-09 17:18:40,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +3: [2023-02-09 17:18:40,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +0: [2023-02-09 17:18:40,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:40,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:40,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:40,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:40,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +5: [2023-02-09 17:18:40,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:40,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt... +1: [2023-02-09 17:18:40,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:40,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:40,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:40,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:40,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:40,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:40,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:40,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:41,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:41,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:41,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:41,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:41,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:41,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:41,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:41,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:41,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:41,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:41,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:41,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:41,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:41,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:41,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:41,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:41,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:41,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:41,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:41,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:41,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:41,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:41,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:41,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:41,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +7: [2023-02-09 17:18:41,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:41,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +3: [2023-02-09 17:18:41,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +5: [2023-02-09 17:18:41,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:41,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:41,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:41,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:41,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:41,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:41,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +4: [2023-02-09 17:18:41,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +2: [2023-02-09 17:18:41,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +1: [2023-02-09 17:18:41,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +6: [2023-02-09 17:18:41,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:41,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:41,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_13-model_00-model_states.pt. +0: [2023-02-09 17:18:41,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +6: [2023-02-09 17:18:41,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +6: [2023-02-09 17:18:41,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +5: [2023-02-09 17:18:41,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +4: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +1: [2023-02-09 17:18:41,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +7: [2023-02-09 17:18:41,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +3: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +0: [2023-02-09 17:18:41,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt... +2: [2023-02-09 17:18:41,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +7: [2023-02-09 17:18:41,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +2: [2023-02-09 17:18:41,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +1: [2023-02-09 17:18:41,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +4: [2023-02-09 17:18:41,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +5: [2023-02-09 17:18:41,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +3: [2023-02-09 17:18:41,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_14-model_00-model_states.pt. +0: [2023-02-09 17:18:41,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +6: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +3: [2023-02-09 17:18:41,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +7: [2023-02-09 17:18:41,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +5: [2023-02-09 17:18:41,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +6: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +4: [2023-02-09 17:18:41,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +0: [2023-02-09 17:18:41,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +2: [2023-02-09 17:18:41,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt... +1: [2023-02-09 17:18:41,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +4: [2023-02-09 17:18:41,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +3: [2023-02-09 17:18:41,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +1: [2023-02-09 17:18:41,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +7: [2023-02-09 17:18:41,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +5: [2023-02-09 17:18:41,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +2: [2023-02-09 17:18:41,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_15-model_00-model_states.pt. +0: [2023-02-09 17:18:41,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +1: [2023-02-09 17:18:41,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +4: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +2: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +7: [2023-02-09 17:18:41,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +6: [2023-02-09 17:18:41,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +3: [2023-02-09 17:18:41,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +0: [2023-02-09 17:18:41,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt... +5: [2023-02-09 17:18:41,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +5: [2023-02-09 17:18:41,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +1: [2023-02-09 17:18:41,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +7: [2023-02-09 17:18:41,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +3: [2023-02-09 17:18:41,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +6: [2023-02-09 17:18:41,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +4: [2023-02-09 17:18:41,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +2: [2023-02-09 17:18:41,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_16-model_00-model_states.pt. +0: [2023-02-09 17:18:41,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +1: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +6: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +4: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +2: [2023-02-09 17:18:41,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +5: [2023-02-09 17:18:41,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +0: [2023-02-09 17:18:41,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +7: [2023-02-09 17:18:41,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt... +3: [2023-02-09 17:18:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +3: [2023-02-09 17:18:41,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +1: [2023-02-09 17:18:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +6: [2023-02-09 17:18:41,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +7: [2023-02-09 17:18:41,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +5: [2023-02-09 17:18:41,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +4: [2023-02-09 17:18:41,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +2: [2023-02-09 17:18:41,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_17-model_00-model_states.pt. +0: [2023-02-09 17:18:41,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +2: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +1: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +5: [2023-02-09 17:18:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +6: [2023-02-09 17:18:41,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +7: [2023-02-09 17:18:41,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +4: [2023-02-09 17:18:41,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:41,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +0: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +3: [2023-02-09 17:18:41,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:41,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:41,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:41,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt... +4: [2023-02-09 17:18:41,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:41,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:41,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:41,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:41,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +2: [2023-02-09 17:18:41,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:41,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +7: [2023-02-09 17:18:41,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:41,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:41,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:41,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +3: [2023-02-09 17:18:41,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:41,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:41,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:41,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +5: [2023-02-09 17:18:41,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +6: [2023-02-09 17:18:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:41,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +1: [2023-02-09 17:18:41,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:41,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:41,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:41,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:41,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:41,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_18-model_00-model_states.pt. +0: [2023-02-09 17:18:42,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:42,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +6: [2023-02-09 17:18:42,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +1: [2023-02-09 17:18:42,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +3: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +5: [2023-02-09 17:18:42,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +7: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:42,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +4: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +0: [2023-02-09 17:18:42,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt... +2: [2023-02-09 17:18:42,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +4: [2023-02-09 17:18:42,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +3: [2023-02-09 17:18:42,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +5: [2023-02-09 17:18:42,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +1: [2023-02-09 17:18:42,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +6: [2023-02-09 17:18:42,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +2: [2023-02-09 17:18:42,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +7: [2023-02-09 17:18:42,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_19-model_00-model_states.pt. +0: [2023-02-09 17:18:42,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +6: [2023-02-09 17:18:42,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +4: [2023-02-09 17:18:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +2: [2023-02-09 17:18:42,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +7: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +0: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +3: [2023-02-09 17:18:42,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +5: [2023-02-09 17:18:42,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt... +1: [2023-02-09 17:18:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +3: [2023-02-09 17:18:42,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +7: [2023-02-09 17:18:42,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +1: [2023-02-09 17:18:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +5: [2023-02-09 17:18:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +4: [2023-02-09 17:18:42,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +6: [2023-02-09 17:18:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +2: [2023-02-09 17:18:42,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_20-model_00-model_states.pt. +0: [2023-02-09 17:18:42,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +4: [2023-02-09 17:18:42,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +3: [2023-02-09 17:18:42,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +6: [2023-02-09 17:18:42,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +2: [2023-02-09 17:18:42,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +1: [2023-02-09 17:18:42,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +6: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +0: [2023-02-09 17:18:42,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +6: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +5: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +5: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt... +7: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +5: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +5: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +3: [2023-02-09 17:18:42,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +4: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +1: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +7: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +7: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +3: [2023-02-09 17:18:42,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +3: [2023-02-09 17:18:42,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: > overriding learning rate value to 0.0002 +0: > overriding minimum learning rate value to 2e-05 +0: > overriding warmup iterations value to 0 +0: > overriding total number of iterations value to 1 +0: > overriding decay style value to cosine +1: [2023-02-09 17:18:42,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +1: [2023-02-09 17:18:42,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +1: [2023-02-09 17:18:42,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +4: [2023-02-09 17:18:42,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +4: [2023-02-09 17:18:42,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +2: [2023-02-09 17:18:42,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +2: [2023-02-09 17:18:42,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +2: [2023-02-09 17:18:42,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_21-model_00-model_states.pt. +0: [2023-02-09 17:18:42,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +0: [2023-02-09 17:18:42,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt... +0: [2023-02-09 17:18:42,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/layer_23-model_00-model_states.pt. +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +6: [2023-02-09 17:18:43,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... +4: [2023-02-09 17:18:43,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:43,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +5: [2023-02-09 17:18:44,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +2: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... +1: [2023-02-09 17:18:44,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +7: [2023-02-09 17:18:44,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +3: [2023-02-09 17:18:44,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... +0: [2023-02-09 17:18:44,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0 +5: [2023-02-09 17:18:44,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46 +6: [2023-02-09 17:18:44,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,328] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54 +5: [2023-02-09 17:18:44,329] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46 +0: [2023-02-09 17:18:44,329] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0 +0: could not find arguments in the checkpoint ... +0: checkpoint version 3.0 +6: [2023-02-09 17:18:44,331] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54 +0: [2023-02-09 17:18:44,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,348] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7 +0: [2023-02-09 17:18:44,352] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7 +3: [2023-02-09 17:18:44,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,374] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29 +3: [2023-02-09 17:18:44,378] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29 +6: [2023-02-09 17:18:44,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,389] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50 +5: [2023-02-09 17:18:44,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,391] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47 +6: [2023-02-09 17:18:44,392] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50 +5: [2023-02-09 17:18:44,394] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47 +2: [2023-02-09 17:18:44,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:44,409] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16 +2: [2023-02-09 17:18:44,413] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16 +1: [2023-02-09 17:18:44,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,414] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9 +4: [2023-02-09 17:18:44,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,418] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36 +1: [2023-02-09 17:18:44,418] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9 +4: [2023-02-09 17:18:44,421] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36 +2: [2023-02-09 17:18:44,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,428] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52 +2: [2023-02-09 17:18:44,428] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23 +6: [2023-02-09 17:18:44,432] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52 +2: [2023-02-09 17:18:44,433] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23 +6: [2023-02-09 17:18:44,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,444] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49 +6: [2023-02-09 17:18:44,449] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49 +6: [2023-02-09 17:18:44,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,461] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53 +6: [2023-02-09 17:18:44,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,462] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51 +6: [2023-02-09 17:18:44,465] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53 +6: [2023-02-09 17:18:44,468] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51 +5: [2023-02-09 17:18:44,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45 +7: [2023-02-09 17:18:44,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,477] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60 +5: [2023-02-09 17:18:44,479] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45 +7: [2023-02-09 17:18:44,481] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60 +7: [2023-02-09 17:18:44,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,487] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56 +7: [2023-02-09 17:18:44,491] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56 +7: [2023-02-09 17:18:44,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,491] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61 +1: [2023-02-09 17:18:44,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,494] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14 +7: [2023-02-09 17:18:44,495] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61 +1: [2023-02-09 17:18:44,498] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14 +2: [2023-02-09 17:18:44,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:44,502] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19 +2: [2023-02-09 17:18:44,506] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19 +4: [2023-02-09 17:18:44,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,510] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38 +4: [2023-02-09 17:18:44,514] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38 +3: [2023-02-09 17:18:44,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,518] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26 +7: [2023-02-09 17:18:44,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,518] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59 +3: [2023-02-09 17:18:44,522] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26 +7: [2023-02-09 17:18:44,522] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59 +4: [2023-02-09 17:18:44,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,525] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33 +5: [2023-02-09 17:18:44,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,526] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44 +5: [2023-02-09 17:18:44,530] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44 +4: [2023-02-09 17:18:44,531] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33 +1: [2023-02-09 17:18:44,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,538] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13 +1: [2023-02-09 17:18:44,542] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13 +7: [2023-02-09 17:18:44,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,549] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63 +7: [2023-02-09 17:18:44,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,549] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58 +7: [2023-02-09 17:18:44,554] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58 +7: [2023-02-09 17:18:44,554] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63 +1: [2023-02-09 17:18:44,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,555] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12 +1: [2023-02-09 17:18:44,559] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12 +1: [2023-02-09 17:18:44,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,574] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15 +5: [2023-02-09 17:18:44,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,576] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42 +1: [2023-02-09 17:18:44,578] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15 +5: [2023-02-09 17:18:44,580] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42 +6: [2023-02-09 17:18:44,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,587] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48 +3: [2023-02-09 17:18:44,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,590] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31 +6: [2023-02-09 17:18:44,591] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48 +3: [2023-02-09 17:18:44,595] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31 +0: [2023-02-09 17:18:44,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,603] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2 +0: [2023-02-09 17:18:44,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2 +7: [2023-02-09 17:18:44,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,621] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62 +7: [2023-02-09 17:18:44,625] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62 +5: [2023-02-09 17:18:44,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,630] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43 +2: [2023-02-09 17:18:44,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:44,633] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22 +5: [2023-02-09 17:18:44,634] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43 +2: [2023-02-09 17:18:44,636] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22 +3: [2023-02-09 17:18:44,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,644] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27 +5: [2023-02-09 17:18:44,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,645] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40 +1: [2023-02-09 17:18:44,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,645] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11 +3: [2023-02-09 17:18:44,648] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27 +1: [2023-02-09 17:18:44,649] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11 +5: [2023-02-09 17:18:44,649] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40 +4: [2023-02-09 17:18:44,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,656] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34 +4: [2023-02-09 17:18:44,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,657] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37 +3: [2023-02-09 17:18:44,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,659] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25 +4: [2023-02-09 17:18:44,660] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34 +4: [2023-02-09 17:18:44,662] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37 +3: [2023-02-09 17:18:44,664] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25 +7: [2023-02-09 17:18:44,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +7: [2023-02-09 17:18:44,675] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57 +7: [2023-02-09 17:18:44,679] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57 +4: [2023-02-09 17:18:44,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,686] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32 +0: [2023-02-09 17:18:44,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,688] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1 +3: [2023-02-09 17:18:44,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,688] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30 +4: [2023-02-09 17:18:44,690] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32 +3: [2023-02-09 17:18:44,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,692] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28 +3: [2023-02-09 17:18:44,693] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30 +0: [2023-02-09 17:18:44,694] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1 +3: [2023-02-09 17:18:44,696] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28 +2: [2023-02-09 17:18:44,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:44,707] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17 +0: [2023-02-09 17:18:44,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,708] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4 +2: [2023-02-09 17:18:44,711] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17 +0: [2023-02-09 17:18:44,713] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4 +4: [2023-02-09 17:18:44,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,719] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39 +4: [2023-02-09 17:18:44,723] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39 +2: [2023-02-09 17:18:44,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:44,755] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18 +2: [2023-02-09 17:18:44,759] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18 +3: [2023-02-09 17:18:44,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +3: [2023-02-09 17:18:44,760] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24 +3: [2023-02-09 17:18:44,765] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24 +4: [2023-02-09 17:18:44,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +4: [2023-02-09 17:18:44,769] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35 +4: [2023-02-09 17:18:44,774] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35 +5: [2023-02-09 17:18:44,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +5: [2023-02-09 17:18:44,781] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41 +0: [2023-02-09 17:18:44,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,782] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6 +5: [2023-02-09 17:18:44,785] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41 +0: [2023-02-09 17:18:44,786] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6 +1: [2023-02-09 17:18:44,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,801] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10 +1: [2023-02-09 17:18:44,805] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10 +6: [2023-02-09 17:18:44,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +6: [2023-02-09 17:18:44,830] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55 +6: [2023-02-09 17:18:44,834] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55 +2: [2023-02-09 17:18:44,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:44,862] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21 +2: [2023-02-09 17:18:44,865] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21 +1: [2023-02-09 17:18:44,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +1: [2023-02-09 17:18:44,870] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8 +0: [2023-02-09 17:18:44,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,874] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5 +1: [2023-02-09 17:18:44,874] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8 +0: [2023-02-09 17:18:44,878] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5 +0: [2023-02-09 17:18:44,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +0: [2023-02-09 17:18:44,893] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3 +0: [2023-02-09 17:18:44,898] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3 +2: [2023-02-09 17:18:44,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_619m22b4b8/global_step41007/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. +2: [2023-02-09 17:18:44,938] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20 +2: [2023-02-09 17:18:44,942] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20 +0: successfully loaded checkpoint from checkpoints_619m22b4b8 at iteration 0 +7: time (ms) | load-checkpoint: 6552.19 +0: estimated model parameters: 0.618714624 +0: estimated model parameters without embeddings: 0.538301952 +0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-09 17:18:45 +0: > building train, validation, and test datasets ... +0: > datasets target sizes (minimum size): +0: train: 1 +0: validation: 25600 +0: test: 25600 +0: > building train, validation, and test datasets for GPT ... +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.037315 seconds +0: number of documents: 10028711 +0: > dataset split: +0: train: +0: document indices in [0, 10028711) total of 10028711 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.096 seconds +0: total number of samples: 2342263 +0: total number of epochs: 1 +0: > building dataset index ... +0: reading sizes... +0: reading pointers... +0: reading document index... +0: creating numpy buffer of mmap... +0: creating memory view of numpy buffer... +0: > finished creating indexed dataset in 0.038208 seconds +0: number of documents: 364608 +0: > dataset split: +0: validation: +0: document indices in [0, 364608) total of 364608 documents +0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy +0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy +0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy +0: loaded indexed file in 0.078 seconds +0: total number of samples: 84978 +0: total number of epochs: 1 +0: > finished creating GPT datasets ... +0: [after dataloaders are built] datetime: 2023-02-09 17:19:00 +0: done with setup ... +0: training ... +7: time (ms) | model-and-optimizer-setup: 24523.17 | train/valid/test-data-iterators-setup: 14233.84 +0: [after training is done] datetime: 2023-02-09 17:19:00 +7: ----------------------------------------------------------------------------------------------------------------- +7: validation loss at the end of training for val data | lm loss value: 2.912009E+00 | lm loss PPL: 1.839372E+01 | +7: ----------------------------------------------------------------------------------------------------------------- +END 2817194: Thu 09 Feb 2023 05:19:39 PM EET diff --git a/619m22b4b8/sbatch_619m22b4b8.sh b/619m22b4b8/sbatch_619m22b4b8.sh new file mode 100755 index 0000000000000000000000000000000000000000..c22f8e8a363fcadfd0a8c6795196f9ab2e5c4905 --- /dev/null +++ b/619m22b4b8/sbatch_619m22b4b8.sh @@ -0,0 +1,163 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m22b4b8 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train4b8.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 21500000000 +# -> Samples: 10498047 +TRAIN_SAMPLES=10_498_047 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 104_980 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 1 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m22b4b8/sbatch_619m22b4b8val.sh b/619m22b4b8/sbatch_619m22b4b8val.sh new file mode 100644 index 0000000000000000000000000000000000000000..cd7a8be6049cd6135914bed5f48abeca855819e5 --- /dev/null +++ b/619m22b4b8/sbatch_619m22b4b8val.sh @@ -0,0 +1,167 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=619m22b4b8val +VARIANT_CKPT=619m22b4b8 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train4b8.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_4B8_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=1 + +MICRO_BATCH_SIZE=4 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_632M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 21500000000 +# -> Samples: 10498047 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --override-lr-scheduler \ + --no-load-optim \ + --reset-progress \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/619m22b4b8/tensorboard_619m22b4b8/events.out.tfevents.1675528981.nid006802.9340.0 b/619m22b4b8/tensorboard_619m22b4b8/events.out.tfevents.1675528981.nid006802.9340.0 new file mode 100644 index 0000000000000000000000000000000000000000..31ec651ebda3f1f22a5b31b82a6985203ec8f294 --- /dev/null +++ b/619m22b4b8/tensorboard_619m22b4b8/events.out.tfevents.1675528981.nid006802.9340.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1248aa93090623b13bc415e7c09edfc6abda45df18634004812e444f611c72 +size 73168244 diff --git a/619m22b4b8/tensorboard_619m22b4b8/events.out.tfevents.1675874501.nid007115.89758.0 b/619m22b4b8/tensorboard_619m22b4b8/events.out.tfevents.1675874501.nid007115.89758.0 new file mode 100644 index 0000000000000000000000000000000000000000..afd30fa794311bfc175b48cf9ad91eb7500853dc --- /dev/null +++ b/619m22b4b8/tensorboard_619m22b4b8/events.out.tfevents.1675874501.nid007115.89758.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:683649735f84b6a57b86825af82f776d31338a8553218ea6ff3e4b86195f1eb8 +size 1825448 diff --git a/619m22b4b8/tensorboard_619m22b4b8val/events.out.tfevents.1675955854.nid006946.62402.0 b/619m22b4b8/tensorboard_619m22b4b8val/events.out.tfevents.1675955854.nid006946.62402.0 new file mode 100644 index 0000000000000000000000000000000000000000..4277140ae969b9e44e10ed2487b7bceb7a07b55e --- /dev/null +++ b/619m22b4b8/tensorboard_619m22b4b8val/events.out.tfevents.1675955854.nid006946.62402.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4141bd96b03aa11a8173e5c3220d3fde3f608b3bb67998fc6fccb38143d5f6 +size 980 diff --git a/619m22b4b8/transformers/config.json b/619m22b4b8/transformers/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a1592eaab83db706537cb6afd4da42ec9828de43 --- /dev/null +++ b/619m22b4b8/transformers/config.json @@ -0,0 +1 @@ +{"vocab_size": 50304, "n_positions": 2048, "n_embd": 1536, "n_layer": 19, "n_head": 12, "n_inner": 6144, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"} \ No newline at end of file diff --git a/619m22b4b8/transformers/pytorch_model.bin b/619m22b4b8/transformers/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..ccf70aacc17803bf3a8b54fdc76886a7e5567362 --- /dev/null +++ b/619m22b4b8/transformers/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73fa99f32e9dcf05af664ce9361ac0cd9518f33a60d89dcbe4ba7063389fc98c +size 1396896901